home *** CD-ROM | disk | FTP | other *** search
/ Enter 2004 April / enter-2004-04.iso / files / httrack-3.30.exe / {app} / src / htsback.c < prev    next >
Encoding:
C/C++ Source or Header  |  2003-10-11  |  112.5 KB  |  2,898 lines
  1. /* ------------------------------------------------------------ */
  2. /*
  3. HTTrack Website Copier, Offline Browser for Windows and Unix
  4. Copyright (C) Xavier Roche and other contributors
  5.  
  6. This program is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU General Public License
  8. as published by the Free Software Foundation; either version 2
  9. of the License, or any later version.
  10.  
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. GNU General Public License for more details.
  15.  
  16. You should have received a copy of the GNU General Public License
  17. along with this program; if not, write to the Free Software
  18. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19.  
  20.  
  21. Important notes:
  22.  
  23. - We hereby ask people using this source NOT to use it in purpose of grabbing
  24. emails addresses, or collecting any other private information on persons.
  25. This would disgrace our work, and spoil the many hours we spent on it.
  26.  
  27.  
  28. Please visit our Website: http://www.httrack.com
  29. */
  30.  
  31.  
  32. /* ------------------------------------------------------------ */
  33. /* File: httrack.c subroutines:                                 */
  34. /*       backing system (multiple socket download)              */
  35. /* Author: Xavier Roche                                         */
  36. /* ------------------------------------------------------------ */
  37.  
  38. #include "htsback.h"
  39.  
  40. /* specific definitions */
  41. #include "htsbase.h"
  42. #include "htsnet.h"
  43. #include "htsthread.h"
  44. #include <stdio.h>
  45. #include <stdlib.h>
  46. #include <string.h>
  47. #include <time.h>
  48. /* END specific definitions */
  49.  
  50. //#if HTS_WIN
  51. #include "htsftp.h"
  52. #if HTS_USEZLIB
  53. #include "htszlib.h"
  54. #endif
  55. //#endif
  56.  
  57. #if HTS_WIN
  58. #ifndef __cplusplus
  59. // DOS
  60. #include <process.h>    /* _beginthread, _endthread */
  61. #endif
  62. #else
  63. #endif
  64.  
  65. #undef test_flush
  66. #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->errlog) { fflush(opt->errlog);  } }
  67.  
  68. #define VT_CLREOL       "\33[K"
  69.  
  70.  
  71. // ---
  72. // routines de backing
  73. // retourne l'index d'un lien dans un tableau de backing
  74. int back_index(lien_back* back,int back_max,char* adr,char* fil,char* sav) {
  75.   int i=0;
  76.   int index=-1;
  77.   while( i<back_max ) {
  78.     if (back[i].status>=0)    // rΘception OU prΩt
  79.       if (strfield2(back[i].url_adr,adr)) {
  80.         if (strcmp(back[i].url_fil,fil)==0) {
  81.           if (index==-1)    /* first time we meet, store it */
  82.             index=i;
  83.           else if (strcmp(back[i].url_sav,sav)==0) {  /* oops, check sav too */
  84.             index=i;
  85.             return index;
  86.           }
  87.         }
  88.       }
  89.     i++;
  90.   }
  91.   return index;
  92. }
  93.  
  94. // nombre d'entrΘes libres dans le backing
  95. int back_available(lien_back* back,int back_max) {
  96.   int i;
  97.   int nb=0;
  98.   for(i=0;i<back_max;i++)
  99.     if (back[i].status==-1)     /* libre */
  100.       nb++;
  101.   return nb;
  102. }
  103.  
  104. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  105. LLint back_incache(lien_back* back,int back_max) {
  106.   int i;
  107.   LLint sum=0;
  108.   for(i=0;i<back_max;i++)
  109.     if (back[i].status!=-1)
  110.       if (back[i].r.adr)       // ne comptabilier que les blocs en mΘmoire
  111.         sum+=max(back[i].r.size,back[i].r.totalsize);
  112.   return sum;
  113. }
  114.  
  115. // le lien a-t-il ΘtΘ mis en backing?
  116. HTS_INLINE int back_exist(lien_back* back,int back_max,char* adr,char* fil,char* sav) {
  117.   return (back_index(back,back_max,adr,fil,sav)>=0);
  118. }
  119.  
  120. // nombre de sockets en tΓche de fond
  121. int back_nsoc(lien_back* back,int back_max) {
  122.   int n=0;
  123.   int i;
  124.   for(i=0;i<back_max;i++)
  125.     if (back[i].status > 0)    // only receive
  126.       n++;
  127.  
  128.   return n;
  129. }
  130. int back_nsoc_overall(lien_back* back,int back_max) {
  131.   int n=0;
  132.   int i;
  133.   for(i=0;i<back_max;i++)
  134.     if (back[i].status > 0 || back[i].status == -103)
  135.       n++;
  136.  
  137.   return n;
  138. }
  139.  
  140. // objet (lien) tΘlΘchargΘ ou transfΘrΘ depuis le cache
  141. //
  142. // fermer les paramΦtres de transfert,
  143. // et notamment vΘrifier les fichiers compressΘs (dΘcompresser), callback etc.
  144. int back_finalize(httrackp* opt,cache_back* cache,lien_back* back,int p) {
  145.   if (
  146.       (back[p].status == 0)      // ready
  147.       &&
  148.       (!back[p].testmode)        // not test mode
  149.       &&
  150.       (back[p].r.statuscode>0)   // not internal error
  151.       ) {
  152.     char* state="unknown";
  153.    
  154.     /* dΘcompression */
  155. #if HTS_USEZLIB
  156.     if (gz_is_available && back[p].r.compressed) {
  157.       if (back[p].r.size > 0) {
  158.         //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
  159.         // stats
  160.         back[p].compressed_size=back[p].r.size;
  161.         // en mΘmoire -> passage sur disque
  162.         if (!back[p].r.is_write) {
  163.           back[p].tmpfile_buffer[0]='\0';
  164.           back[p].tmpfile=tmpnam(back[p].tmpfile_buffer);
  165.           if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  166.             back[p].r.out=fopen(back[p].tmpfile,"wb");
  167.             if (back[p].r.out) {
  168.               if ((back[p].r.adr) && (back[p].r.size>0)) {
  169.                 if (fwrite(back[p].r.adr,1,(INTsys)back[p].r.size,back[p].r.out) != back[p].r.size) {
  170.                   back[p].r.statuscode=-1;
  171.                   strcpybuff(back[p].r.msg,"Write error when decompressing");
  172.                 }
  173.               } else {
  174.                 back[p].tmpfile[0]='\0';
  175.                 back[p].r.statuscode=-1;
  176.                 strcpybuff(back[p].r.msg,"Empty compressed file");
  177.               }
  178.             } else {
  179.               back[p].tmpfile[0]='\0';
  180.               back[p].r.statuscode=-1;
  181.               strcpybuff(back[p].r.msg,"Open error when decompressing");
  182.             }
  183.           }
  184.         }
  185.         // fermer fichier sortie
  186.         if (back[p].r.out!=NULL) {
  187.           fclose(back[p].r.out);
  188.           back[p].r.out=NULL;
  189.         }
  190.         // dΘcompression
  191.         if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0' && back[p].url_sav[0]) {
  192.           LLint size;
  193.           filecreateempty(back[p].url_sav);      // filenote & co
  194.           if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) {
  195.             back[p].r.size=back[p].r.totalsize=size;
  196.             // fichier -> mΘmoire
  197.             if (!back[p].r.is_write) {
  198.               deleteaddr(&back[p].r);
  199.               back[p].r.adr=readfile(back[p].url_sav);
  200.               if (!back[p].r.adr) {
  201.                 back[p].r.statuscode=-1;
  202.                 strcpybuff(back[p].r.msg,"Read error when decompressing");
  203.               }
  204.               remove(back[p].url_sav);
  205.             }
  206.           }
  207.           remove(back[p].tmpfile);
  208.         }
  209.         // stats
  210.         HTS_STAT.total_packed+=back[p].compressed_size;
  211.         HTS_STAT.total_unpacked+=back[p].r.size;
  212.         HTS_STAT.total_packedfiles++;
  213.         // unflag
  214.       }
  215.     }
  216.     back[p].r.compressed=0;
  217. #endif
  218.     
  219.     /* Stats */
  220.     if (cache->txt) {
  221.       char flags[32];
  222.       char s[256];
  223.       time_t tt;
  224.       struct tm* A;
  225.       tt=time(NULL);
  226.       A=localtime(&tt);
  227.       if (A == NULL) {
  228.         int localtime_returned_null=0;
  229.         assert(localtime_returned_null);
  230.       }
  231.       strftime(s,250,"%H:%M:%S",A);
  232.       
  233.       flags[0]='\0';
  234.       /* input flags */
  235.       if (back[p].is_update)
  236.         strcatbuff(flags, "U");   // update request
  237.       else
  238.         strcatbuff(flags, "-");
  239.       if (back[p].range_req_size)
  240.         strcatbuff(flags, "R");   // range request
  241.       else
  242.         strcatbuff(flags, "-");
  243.       /* state flags */
  244.       if (back[p].r.is_file)  // direct to disk
  245.         strcatbuff(flags, "F");
  246.       else
  247.         strcatbuff(flags, "-");
  248.       /* output flags */
  249.       if (!back[p].r.notmodified)
  250.         strcatbuff(flags, "M");   // modified
  251.       else
  252.         strcatbuff(flags, "-");
  253.       if (back[p].r.is_chunk)  // chunked
  254.         strcatbuff(flags, "C");
  255.       else
  256.         strcatbuff(flags, "-");
  257.       if (back[p].r.compressed)
  258.         strcatbuff(flags, "Z");   // gzip
  259.       else
  260.         strcatbuff(flags, "-");
  261.       /* Err I had to split these.. */
  262.       fprintf(cache->txt,"%s\t", s);
  263.       fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size);
  264.       fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize);
  265.       fprintf(cache->txt,"\t%s\t",flags);
  266.     }
  267.     if (back[p].r.statuscode==200) {
  268.       if (back[p].r.size>=0) {
  269.         if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) {
  270.           HTS_STAT.stat_bytes+=back[p].r.size;
  271.           HTS_STAT.stat_files++;
  272.         }
  273.         if ( (!back[p].r.notmodified) && (opt->is_update) ) { 
  274.           HTS_STAT.stat_updated_files++;       // page modifiΘe
  275.           if (opt->log!=NULL) {
  276.             fspc(opt->log,"info");
  277.             if (back[p].is_update) {
  278.               fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  279.             } else {
  280.               fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  281.             }
  282.             test_flush;
  283.           }
  284.           if (cache->txt) {
  285.             if (back[p].is_update) {
  286.               state="updated";
  287.             } else {
  288.               state="added";
  289.             }
  290.           }
  291.         } else {
  292.           if ( (opt->debug>0) && (opt->log!=NULL) ) {
  293.             fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  294.             test_flush;
  295.           }
  296.           if (cache->txt) {
  297.             if (opt->is_update)
  298.               state="untouched";
  299.             else
  300.               state="added";
  301.           }
  302.         }
  303.       } else {
  304.         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  305.           fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  306.           test_flush;
  307.         }
  308.         if (cache->txt) {
  309.           state="empty";
  310.         }
  311.       }
  312.     } else {
  313.       if ( (opt->debug>0) && (opt->log!=NULL) ) {
  314.         fspc(opt->log,"info"); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  315.       }
  316.       if (cache->txt) {
  317.         state="error";
  318.       }
  319.     }
  320.     if (cache->txt) {
  321.       fprintf(cache->txt,
  322.         "%d\t"
  323.         "%s ('%s')\t"
  324.         "%s\t"
  325.         "%s%s\t"
  326.         "%s%s\t%s\t"
  327.         "(from %s%s)"
  328.         LF,
  329.         back[p].r.statuscode,
  330.         state, escape_check_url_addr(back[p].r.msg),
  331.         escape_check_url_addr(back[p].r.contenttype),
  332.         ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr((back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)),
  333.         escape_check_url_addr(back[p].url_adr),escape_check_url_addr(back[p].url_fil),escape_check_url_addr(back[p].url_sav),
  334.         escape_check_url_addr(back[p].referer_adr),escape_check_url_addr(back[p].referer_fil)
  335.         );
  336.       if (opt->flush)
  337.         fflush(cache->txt);
  338.     }
  339.     
  340.     /* Cache */
  341.     cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  342.  
  343.     // status finished callback
  344. #if HTS_ANALYSTE
  345.     hts_htmlcheck_xfrstatus(&back[p]);
  346. #endif
  347.     return 0;
  348.   }
  349.   return -1;
  350. }
  351.  
  352. /* try to keep the connection alive */
  353. int back_letlive(httrackp* opt, lien_back* back, int p) {
  354.   htsblk* src = &back[p].r;
  355.   if (src && !src->is_file 
  356.     && src->soc != INVALID_SOCKET
  357.     && src->statuscode >= 0           /* no timeout errors & co */
  358.     && src->keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  359.     && !check_sockerror(src->soc)
  360.     /*&& !check_sockdata(src->soc)*/     /* no unexpected data */
  361.     ) {
  362.     htsblk tmp;
  363.     memset(&tmp, 0, sizeof(tmp));
  364.     /* clear everything but connection: switch, close, and reswitch */
  365.     back_connxfr(src, &tmp);
  366.     back_delete(opt, back, p);
  367.     //deletehttp(src);
  368.     back_connxfr(&tmp, src);
  369.     src->req.flush_garbage=1;     /* ignore CRLF garbage */
  370.     return 1;
  371.   }
  372.   return 0;
  373. }
  374.  
  375. void back_connxfr(htsblk* src, htsblk* dst) {
  376.   dst->soc = src->soc;
  377.   src->soc = INVALID_SOCKET;
  378. #if HTS_USEOPENSSL
  379.   dst->ssl = src->ssl;
  380.   src->ssl = 0;
  381.   dst->ssl_con = src->ssl_con;
  382.   src->ssl_con = NULL;
  383. #endif
  384.   dst->keep_alive = src->keep_alive;
  385.   src->keep_alive = 0;
  386.   dst->keep_alive_max = src->keep_alive_max;
  387.   src->keep_alive_max = 0;
  388.   dst->keep_alive_t = src->keep_alive_t;
  389.   src->keep_alive_t = 0;
  390.   dst->debugid = src->debugid;
  391.   src->debugid = 0;
  392. }
  393.  
  394. // clear, or leave for keep-alive
  395. int back_maydelete(httrackp* opt,lien_back* back, int p) {
  396.   if (p>=0) {    // on sait jamais..
  397.     if (!opt->nokeepalive
  398.       && back[p].r.keep_alive 
  399.       && back[p].r.keep_alive_max > 1
  400.       && back[p].ka_time_start 
  401.       && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t
  402.       ) {
  403.       lien_back tmp;
  404.       strcpybuff(tmp.url_adr, back[p].url_adr);
  405.       if (back_letlive(opt, back, p)) {
  406.         strcpybuff(back[p].url_adr, tmp.url_adr);
  407.         back[p].status = -103;  // alive & waiting
  408.         if ((opt->debug>1) && (opt->log!=NULL)) {
  409.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully saved #%d (%s)"LF, 
  410.             back[p].r.debugid,
  411.             back[p].url_adr); test_flush;
  412.         }
  413.         return 1;
  414.       }
  415.     }
  416.     back_delete(opt,back, p);
  417.   }
  418.   return 0;
  419. }
  420.  
  421. // clear, or leave for keep-alive
  422. void back_maydeletehttp(httrackp* opt, lien_back* back, int back_max, int p) {
  423.   if (back[p].r.soc!=INVALID_SOCKET) {
  424.     int q;
  425.     if (!opt->nokeepalive
  426.       && back[p].r.keep_alive 
  427.       && back[p].r.keep_alive_max > 1
  428.       && back[p].ka_time_start 
  429.       && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t
  430.       && ( q = back_search(opt, back, back_max) ) >= 0
  431.       ) 
  432.     {
  433.       lien_back tmp;
  434.       strcpybuff(tmp.url_adr, back[p].url_adr);
  435.       deletehttp(&back[q].r);               // security check
  436.       back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q
  437.       back[q].ka_time_start = back[p].ka_time_start;  // refresh
  438.       back[p].r.soc = INVALID_SOCKET;
  439.       strcpybuff(back[q].url_adr, tmp.url_adr); // address
  440.       back[q].status = -103;  // alive & waiting
  441.       if ((opt->debug>1) && (opt->log!=NULL)) {
  442.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully preserved #%d (%s)"LF, 
  443.           back[q].r.debugid,
  444.           back[q].url_adr); test_flush;
  445.       }
  446.     } else {
  447.       deletehttp(&back[p].r);
  448.       back[p].r.soc = INVALID_SOCKET;
  449.     }
  450.   }
  451. }
  452.  
  453.  
  454. /* attempt to attach a live connection to this slot */
  455. int back_trylive(httrackp* opt,lien_back* back, int back_max, int p) {
  456.   if (p>=0 && back[p].status != -103) {     // we never know..
  457.     int i = back_searchlive(opt,back, back_max, back[p].url_adr);   // search slot
  458.     if (i >= 0 && i != p) {
  459.       deletehttp(&back[p].r);               // security check
  460.       back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p
  461.       back_delete(opt,back, i);                 // delete old slot
  462.       back[p].status=100;                   // ready to connect
  463.       return 1;                             // success: will reuse live connection
  464.     }
  465.   }
  466.   return 0;
  467. }
  468.  
  469. /* search for a live position, or, if not possible, try to return a new one */
  470. int back_searchlive(httrackp* opt, lien_back* back, int back_max, char* search_addr) {
  471.   int i;
  472.  
  473.   /* search for a live socket */
  474.   for(i = 0 ; i < back_max ; i++ ) {
  475.     if (back[i].status == -103) {
  476.       if (strfield2(back[i].url_adr, search_addr)) {   /* same location (xxc: check also virtual hosts?) */
  477.         if (time_local() < back[i].ka_time_start + back[i].r.keep_alive_t) {
  478.           return i;
  479.         }
  480.       }
  481.     }
  482.   }
  483.   return -1;
  484. }
  485.   
  486. int back_search(httrackp* opt,lien_back* back, int back_max) {
  487.   int i;
  488.  
  489.   /* try to find an empty place */
  490.   for(i = 0 ; i < back_max ; i++ ) {
  491.     if (back[i].status == -1) {
  492.       return i;
  493.     }
  494.   }
  495.  
  496.   /* couldn't find an empty place, try to requisition a keep-alive place */
  497.   for(i = 0 ; i < back_max ; i++ ) {
  498.     if (back[i].status == -103) {
  499.       /* close this place */
  500.       back_delete(opt,back, i);
  501.       return i;
  502.     }
  503.   }
  504.  
  505.   /* oops, can't find a place */
  506.   return -1;
  507. }
  508.  
  509. // effacer entrΘe
  510. int back_delete(httrackp* opt, lien_back* back, int p) {
  511.   if (p>=0) {    // on sait jamais..
  512.     // VΘrificateur d'intΘgritΘ
  513.     #if DEBUG_CHECKINT
  514.     _CHECKINT(&back[p],"Appel back_delete")
  515.     #endif
  516. #if HTS_DEBUG_CLOSESOCK
  517.     char info[256];
  518.     sprintf(info,"back_delete: #%d\n",p);
  519.     DEBUG_W2(info);
  520. #endif
  521.  
  522.     // LibΘrer tous les sockets, handles, buffers..
  523.     if (back[p].r.soc!=INVALID_SOCKET) {
  524. #if HTS_DEBUG_CLOSESOCK
  525.       DEBUG_W("back_delete: deletehttp\n");
  526. #endif
  527.       deletehttp(&back[p].r);
  528.       back[p].r.soc=INVALID_SOCKET;
  529.     }
  530.     
  531.     if (back[p].r.adr!=NULL) {  // reste un bloc α dΘsallouer
  532.       freet(back[p].r.adr);
  533.       back[p].r.adr=NULL;
  534.     }
  535.     if (back[p].chunk_adr!=NULL) {  // reste un bloc α dΘsallouer
  536.       freet(back[p].chunk_adr);
  537.       back[p].chunk_adr=NULL;
  538.       back[p].chunk_size=0;
  539.       back[p].chunk_blocksize=0;
  540.       back[p].is_chunk=0;
  541.     }
  542.     // if (back[p].r.is_file) {  // fermer fichier entrΘe
  543.     if (back[p].r.fp!=NULL) {
  544.       fclose(back[p].r.fp);
  545.       back[p].r.fp=NULL;
  546.     }
  547.     // }
  548.  
  549.     /* fichier de sortie */
  550.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  551.       fclose(back[p].r.out);
  552.       back[p].r.out=NULL;
  553.     }
  554.  
  555.     if (back[p].r.is_write) {     // ecriture directe
  556.       /* Θcrire date "remote" */
  557.       if (strnotempty(back[p].url_sav))          // normalement existe si on a un fichier de sortie
  558.       if (strnotempty(back[p].r.lastmodified))   // last-modified existe
  559.       if (fexist(back[p].url_sav))          // ainsi que le fichier
  560.         set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified);
  561.  
  562.       /* executer commande utilisateur aprΦs chargement du fichier */
  563.       //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil);
  564.       back[p].r.is_write=0;
  565.     }
  566.     
  567.     // Tout nettoyer
  568.     memset(&back[p], 0, sizeof(lien_back));  
  569.     back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  570.     
  571.     // Le plus important: libΘrer le champ
  572.     back[p].status=-1;
  573.   }
  574.   return 0;
  575. }
  576.  
  577. /* Space left on backing stack */
  578. int back_stack_available(lien_back* back,int back_max) {
  579.   int p=0,n=0;
  580.   for( ; p < back_max ; p++ )
  581.     if ( back[p].status == -1 )
  582.       n++;
  583.   return n;
  584. }
  585.  
  586. // ajouter un lien en backing
  587. int back_add(lien_back* back,int back_max,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test,int* pass2_ptr) {
  588.   int p=0;
  589.  
  590.   // vΘrifier cohΘrence de adr et fil (non vide!)
  591.   if (strnotempty(adr)==0) {
  592.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  593.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: adr is empty for back_add"LF);
  594.     }
  595.     return -1;    // erreur!
  596.   }
  597.   if (strnotempty(fil)==0) {
  598.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  599.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: fil is empty for back_add"LF);
  600.     }
  601.     return -1;    // erreur!
  602.   }
  603.   // FIN vΘrifier cohΘrence de adr et fil (non vide!)
  604.  
  605.   // stats
  606.   opt->state.back_add_stats++;
  607.  
  608.   // rechercher emplacement
  609.   back_clean(opt, cache, back, back_max);
  610.   if ( ( p = back_search(opt, back, back_max) ) >= 0) {
  611.     back[p].send_too[0]='\0';  // Θventuels paramΦtres supplΘmentaires α transmettre au serveur
  612.  
  613.     // clear r
  614.     if (back[p].r.soc!=INVALID_SOCKET) {  /* we never know */
  615.       deletehttp(&back[p].r);
  616.     }
  617.     memset(&(back[p].r), 0, sizeof(htsblk)); 
  618.     back[p].r.soc=INVALID_SOCKET; 
  619.     back[p].r.location=back[p].location_buffer;
  620.  
  621.     // crΘer entrΘe
  622.     strcpybuff(back[p].url_adr,adr);
  623.     strcpybuff(back[p].url_fil,fil);
  624.     strcpybuff(back[p].url_sav,save);
  625.     back[p].pass2_ptr=pass2_ptr;
  626.     // copier referer si besoin
  627.     strcpybuff(back[p].referer_adr,"");
  628.     strcpybuff(back[p].referer_fil,"");
  629.     if ((referer_adr) && (referer_fil)) {       // existe
  630.       if ((strnotempty(referer_adr)) && (strnotempty(referer_fil))) {   // non vide
  631.         if (referer_adr[0]!='!') {    // non dΘtruit
  632.           if (strcmp(referer_adr,"file://")) {      // PAS file://
  633.             if (strcmp(referer_adr,"primary")) {      // pas referer 1er lien
  634.               strcpybuff(back[p].referer_adr,referer_adr);
  635.               strcpybuff(back[p].referer_fil,referer_fil);
  636.             }
  637.           }
  638.         }
  639.       }
  640.     }
  641.     // sav ne sert α rien pour le moment
  642.     back[p].r.size=0;                   // rien n'a encore ΘtΘ chargΘ
  643.     back[p].r.adr=NULL;                 // pas de bloc de mΘmoire
  644.     back[p].r.is_write=0;               // α priori stockage en mΘmoire
  645.     back[p].maxfile_html=opt->maxfile_html;
  646.     back[p].maxfile_nonhtml=opt->maxfile_nonhtml;
  647.     back[p].testmode=test;              // mode test?
  648.     if (!opt->http10)                 // option "forcer 1.0" dΘsactivΘe
  649.       back[p].http11=1;               // autoriser http/1.1
  650.     back[p].head_request=0;
  651.     if (strcmp(back[p].url_sav,BACK_ADD_TEST)==0)    // HEAD
  652.       back[p].head_request=1;
  653.     else if (strcmp(back[p].url_sav,BACK_ADD_TEST2)==0)    // test en GET
  654.       back[p].head_request=2;       // test en get
  655.  
  656.     /* Stop requested - abort backing */
  657.     if (opt->state.stop) {
  658.       back[p].r.statuscode=-1;        // fatal
  659.       strcpybuff(back[p].r.msg,"mirror stopped by user");
  660.       back[p].status=0;  // terminΘ
  661.       if ((opt->debug>0) && (opt->log!=NULL)) {
  662.         fspc(opt->log,"warning"); fprintf(opt->log,"File not added due to mirror cancel: %s%s"LF,adr,fil); test_flush;
  663.       }            
  664.       return 0;
  665.     }
  666.  
  667.     // tester cache
  668.     if ((strcmp(adr,"file://"))           /* pas fichier */
  669.       && ( (!test) || (cache->type==1) )   /* cache prioritaire, laisser passer en test! */
  670.       && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) {  // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
  671.       //if ((!test) && (strcmp(adr,"file://")) 
  672.       //if ((!test) && (strncmp(adr,"ftp://",6)) && (strcmp(adr,"file://")) 
  673. #if HTS_FAST_CACHE
  674.       long int hash_pos;
  675.       int hash_pos_return=0;
  676. #else
  677.       char* a=NULL;
  678. #endif
  679. #if HTS_FAST_CACHE
  680.       if (cache->hashtable) { 
  681. #else
  682.       if (cache->use) { 
  683. #endif
  684.         char buff[HTS_URLMAXSIZE*4];
  685. #if HTS_FAST_CACHE
  686.         strcpybuff(buff,adr); strcatbuff(buff,fil);
  687.         hash_pos_return=inthash_read((inthash)cache->hashtable,buff,(long int*)&hash_pos);
  688. #else
  689.         buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n");
  690.         a=strstr(cache->use,buff);
  691. #endif
  692.         
  693.         // Ok, notΘ en cache->. mais bien prΘsent dans le cache ou sur disque?
  694. #if HTS_FAST_CACHE
  695.         if (hash_pos_return) {
  696. #else
  697.         if (a) {
  698. #endif
  699.           if (!test) {      // non mode test
  700. #if HTS_FAST_CACHE
  701.             int pos=hash_pos;
  702. #else
  703.             int pos=-1;
  704.             a+=strlen(buff);
  705.             sscanf(a,"%d",&pos);    // lire position
  706. #endif
  707.             if (pos<0) {    // pas de mise en cache data, vΘrifier existence
  708.               if (fsize(fconv(save)) <= 0) {  // fichier existe pas ou est vide!
  709.                 int found=0;
  710.  
  711.                 /* It is possible that the file has been moved due to changes in build structure */
  712.                 {
  713.                   char previous_save[HTS_URLMAXSIZE*2];
  714.                   previous_save[0] = '\0';
  715.                   back[p].r = cache_readex(opt, cache, adr, fil, NULL, back[p].location_buffer, previous_save, 0);
  716.                   if (previous_save[0] != '\0' && fexist(fconv(previous_save))) {
  717.                     rename(fconv(previous_save), fconv(save));
  718.                     if (fexist(fconv(save))) {
  719.                       found = 1;
  720.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  721.                         fspc(opt->log,"debug"); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush;
  722.                       }
  723.                     } else {
  724.                       if ((opt->debug>0) && (opt->log!=NULL)) {
  725.                         fspc(opt->log,"error"); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush;
  726.                       }
  727.                     }
  728.                   }
  729.                 }
  730.                 
  731.                 if (!found) {
  732. #if HTS_FAST_CACHE
  733.                   hash_pos_return=0;
  734. #else
  735.                   a=NULL;    
  736. #endif
  737.                   // dΘvalider car non prΘsent sur disque dans structure originale!!!
  738.                   // sinon, le fichier est ok α priori, mais on renverra un if-modified-since pour
  739.                   // en Ωtre s√r
  740.                   if (opt->norecatch) {              // tester norecatch
  741.                     if (!fexist(fconv(save))) {  // fichier existe pas mais dΘclarΘ: on l'a effacΘ
  742.                       FILE* fp=fopen(fconv(save),"wb");
  743.                       if (fp) fclose(fp);
  744.                       if (opt->log!=NULL) {
  745.                         fspc(opt->log,"warning"); fprintf(opt->log,"File must have been erased by user, ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  746.                       }
  747.                     }
  748.                   }
  749.                 }
  750.               }
  751.             }
  752.           }
  753.         }
  754.         //
  755.       } else
  756. #if HTS_FAST_CACHE
  757.         hash_pos_return=0;
  758. #else
  759.         a=NULL;
  760. #endif
  761.  
  762.       // Existe pas en cache, ou bien pas de cache prΘsent
  763. #if HTS_FAST_CACHE
  764.       if (hash_pos_return) {  // OK existe en cache (et donnΘes aussi)!
  765. #else
  766.       if (a!=NULL) {  // OK existe en cache (et donnΘes aussi)!
  767. #endif
  768.         if (cache->type==1) {   // cache prioritaire (pas de test if-modified..)
  769.                                // dans ce cas on peut Θgalement lire des rΘponses cachΘes comme 404,302...
  770.           // lire dans le cache
  771.           if (!test)
  772.             back[p].r = cache_read(opt,cache,adr,fil,save, back[p].location_buffer);
  773.           else
  774.             back[p].r = cache_read(opt,cache,adr,fil,NULL, back[p].location_buffer);  // charger en tΩte uniquement du cache
  775.  
  776.           /* ensure correct location buffer set */
  777.           back[p].r.location=back[p].location_buffer;
  778.  
  779.           /* Interdiction taille par le wizard? --> dΘtruire */
  780.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  781.             if (!back_checksize(opt,&back[p],0)) {
  782.               back[p].status=0;  // FINI
  783.               back[p].r.statuscode=-1;
  784.               if (!back[p].testmode)
  785.                 strcpybuff(back[p].r.msg,"Cached file skipped (too big)");
  786.               else
  787.                 strcpybuff(back[p].r.msg,"Test: Cached file skipped  (too big)");
  788.               return 0;
  789.             }
  790.           }
  791.  
  792.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  793.             if ((opt->debug>0) && (opt->log!=NULL)) {
  794.               if (!test) {
  795.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately loaded from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  796.               } else {
  797.                 fspc(opt->log,"debug"); fprintf(opt->log,"File immediately tested from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  798.               }
  799.             }
  800.             back[p].r.notmodified=1;    // fichier non modifiΘ
  801.             back[p].status=0;  // OK prΩt
  802.  
  803.             // finalize transfer
  804.             if (!test) {
  805.               if (back[p].r.statuscode>0) {
  806.                 back_finalize(opt,cache,back,p);
  807.               }
  808.             }
  809.  
  810.             return 0;
  811.           } else {  // erreur
  812.             // effacer r
  813.             memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  814.             // et continuer (chercher le fichier)
  815.           }
  816.           
  817.         } else if (cache->type==2) {    // si en cache, demander de tester If-Modified-Since
  818.           htsblk r;
  819.           cache_header(opt,cache,adr,fil,&r);
  820.  
  821.           /* Interdiction taille par le wizard? */
  822.           {
  823.             LLint save_totalsize=back[p].r.totalsize;
  824.             back[p].r.totalsize=r.totalsize;
  825.             if (!back_checksize(opt,&back[p],1)) {
  826.               r.statuscode = -1;
  827.               //
  828.               back[p].status=0;  // FINI
  829.               deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET;
  830.               if (!back[p].testmode)
  831.                 strcpybuff(back[p].r.msg,"File too big");
  832.               else
  833.                 strcpybuff(back[p].r.msg,"Test: File too big");
  834.               return 0;
  835.             }
  836.             back[p].r.totalsize=save_totalsize;
  837.           }
  838.           
  839.           if (r.statuscode != -1) {
  840.             if (r.statuscode==200) {     // uniquement des 200 (OK)
  841.               if (strnotempty(r.etag)) {  // ETag (RFC2616)
  842.                 /*
  843.                 - If both an entity tag and a Last-Modified value have been
  844.                 provided by the origin server, SHOULD use both validators in
  845.                 cache-conditional requests. This allows both HTTP/1.0 and
  846.                 HTTP/1.1 caches to respond appropriately.
  847.                 */
  848.                 if (strnotempty(r.lastmodified))
  849.                   sprintf(back[p].send_too,"If-None-Match: %s\r\nIf-Modified-Since: %s\r\n",r.etag,r.lastmodified);
  850.                 else
  851.                   sprintf(back[p].send_too,"If-None-Match: %s\r\n",r.etag);
  852.               }
  853.               else if (strnotempty(r.lastmodified))
  854.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",r.lastmodified);
  855.               else if (strnotempty(cache->lastmodified))
  856.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified);
  857.               
  858.               /* this is an update of a file */
  859.               if (strnotempty(back[p].send_too))
  860.                 back[p].is_update=1;
  861.               back[p].r.req.nocompression=1;   /* Do not compress when updating! */
  862.               
  863.             }
  864.             /* else if (strnotempty(cache->lastmodified))
  865.             sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified);
  866.             */
  867.           }
  868. #if DEBUGCA
  869.           printf("..is modified test %s\n",back[p].send_too);
  870. #endif
  871.         } 
  872.         // Okay, pas trouvΘ dans le cache
  873.         // Et si le fichier existe sur disque?
  874.         // Pas dans le cache: fichier n'a pas ΘtΘ transfΘrΘ du tout, donc pas sur disque?
  875.       } else {
  876.         if (fexist(save)) {    // fichier existe? aghl!
  877.           LLint sz=fsize(save);
  878.           // Bon, lα il est possible que le fichier ait ΘtΘ partiellement transfΘrΘ
  879.           // (s'il l'avait ΘtΘ en totalitΘ il aurait ΘtΘ inscrit dans le cache ET existerait sur disque)
  880.           // PAS de If-Modified-Since, on a pas connaissance des donnΘes α la date du cache
  881.           // On demande juste les donnΘes restantes si le date est valide (206), tout sinon (200)
  882.           if ((ishtml(save) != 1) && (ishtml(back[p].url_fil)!=1)) {   // NON HTML (liens changΘs!!)
  883.             if (sz>0) {    // Fichier non vide? (question bΩte, sinon on transfert tout!)
  884.               if (strnotempty(cache->lastmodified)) {     /* pas de If-.. possible */
  885.                 /*if ( (!opt->http10) && (strnotempty(cache->lastmodified)) ) { */    /* ne pas forcer 1.0 */
  886. #if DEBUGCA
  887.                 printf("..if unmodified since %s size "LLintP"\n",cache->lastmodified,(LLint)sz);
  888. #endif
  889.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  890.                   fspc(opt->log,"debug"); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  891.                 }
  892.                 
  893.                 /* impossible - don't have etag or date
  894.                 if (strnotempty(back[p].r.etag)) {  // ETag (RFC2616)
  895.                 sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag);
  896.                 back[p].http11=1;    // En tΩte 1.1
  897.                 } else if (strnotempty(back[p].r.lastmodified)) {
  898.                 sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified);
  899.                 back[p].http11=1;    // En tΩte 1.1
  900.                 } else 
  901.                 */
  902.                 if (strlen(cache->lastmodified)) {
  903.                   sprintf(back[p].send_too,
  904.                     "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n"
  905.                     ,cache->lastmodified,(LLint)sz);
  906.                   back[p].http11=1;    // En tΩte 1.1
  907.                   back[p].range_req_size=sz;
  908.                   back[p].r.req.range_used=1;
  909.                   back[p].r.req.nocompression=1;
  910.                 } else {
  911.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  912.                     fspc(opt->log,"warning"); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  913.                   }
  914.                 }
  915.                 
  916.               } else { 
  917.                 if ((opt->debug>0) && (opt->errlog!=NULL)) {
  918.                   fspc(opt->errlog,"warning");
  919.                   /*
  920.                   if (opt->http10)
  921.                   fprintf(opt->errlog,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil);
  922.                   else
  923.                   */
  924.                   fprintf(opt->errlog,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  925.                   test_flush;
  926.                 }
  927.                 /* Sinon requΩte normale... */
  928.                 back[p].http11=0;
  929.               }
  930.             } else if (opt->norecatch) {              // tester norecatch
  931.               filenote(save,NULL);       // ne pas purger tout de mΩme
  932.               back[p].status=0;  // OK prΩt
  933.               back[p].r.statuscode=-1;  // erreur
  934.               strcpybuff(back[p].r.msg,"Null-size file not recaught");
  935.               return 0;
  936.             }
  937.           } else {
  938.             if ((opt->debug>0) && (opt->errlog!=NULL)) {
  939.               fspc(opt->errlog,"warning");
  940.               fprintf(opt->errlog,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  941.               test_flush;
  942.             }
  943.             /* Sinon requΩte normale... */
  944.             back[p].http11=0;
  945.           }
  946.         }
  947.       }
  948.     }
  949.  
  950.  
  951.     {
  952.       ///htsblk r;   non directement dans la structure-rΘponse!
  953.       T_SOC soc;
  954.       
  955.       // ouvrir liaison, envoyer requΦte
  956.       // ne pas traiter ou recevoir l'en tΩte immΘdiatement
  957.       memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  958.       // recopier proxy
  959.       memcpy(&(back[p].r.req.proxy), &opt->proxy, sizeof(opt->proxy));
  960.       // et user-agent
  961.       strcpybuff(back[p].r.req.user_agent,opt->user_agent);
  962.       strcpybuff(back[p].r.req.lang_iso,opt->lang_iso);
  963.       back[p].r.req.user_agent_send=opt->user_agent_send;
  964.       // et http11
  965.       back[p].r.req.http11=back[p].http11;
  966.       back[p].r.req.nocompression=opt->nocompression;
  967.       back[p].r.req.nokeepalive=opt->nokeepalive;
  968.  
  969.       // mode ftp, court-circuit!
  970.       if (strfield(back[p].url_adr,"ftp://")) {
  971.         if (back[p].testmode) {
  972.           if ((opt->debug>1) && (opt->errlog!=NULL)) {
  973.             fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: forbidden test with ftp link for back_add"LF);
  974.           }
  975.           return -1;    // erreur pas de test permis
  976.         }
  977.         if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gΘrΘe en thread
  978.           back[p].status=1000;   // connexion ftp
  979. #if USE_BEGINTHREAD
  980.           launch_ftp(&(back[p]));
  981. #else
  982.           {
  983.             char nid[32];
  984.             sprintf(nid,"htsftp%d-in_progress.lock",p);
  985.             strcpybuff(back[p].location_buffer,fconcat(opt->path_log,nid));
  986.           }
  987.           launch_ftp(&(back[p]),back[p].location_buffer,opt->exec);
  988. #endif
  989.           return 0;
  990.         }
  991.       }
  992. #if HTS_USEOPENSSL
  993.       else if (SSL_is_available && strfield(back[p].url_adr,"https://")) {        // let's rock
  994.         back[p].r.ssl = 1;
  995.         // back[p].r.ssl_soc = NULL;
  996.         back[p].r.ssl_con = NULL;
  997.       }
  998. #endif
  999.       
  1000.       if (!back_trylive(opt,back, back_max, p)) {
  1001. #if HTS_XGETHOST
  1002. #if HDEBUG
  1003.         printf("back_solve..\n");
  1004. #endif
  1005.         back[p].status=101;    // tentative de rΘsolution du nom de host
  1006.         soc=INVALID_SOCKET;    // pas encore ouverte
  1007.         back_solve(&back[p]);  // prΘparer
  1008.         if (host_wait(&back[p])) {  // prΩt, par ex fichier ou dispo dans dns
  1009. #if HDEBUG
  1010.           printf("ok, dns cache ready..\n");
  1011. #endif
  1012.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1013.           if (soc==INVALID_SOCKET) {
  1014.             back[p].status=0;  // fini, erreur
  1015.           }
  1016.         }
  1017.         //
  1018. #else
  1019.         //
  1020. #if CNXDEBUG
  1021.         printf("XFopen..\n");
  1022. #endif
  1023.         
  1024.         if (strnotempty(back[p].send_too))    // envoyer un if-modified-since
  1025. #if HTS_XCONN
  1026.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1027. #else
  1028.         soc=http_xfopen(0,0,1,back[p].send_too,adr,fil,&(back[p].r));
  1029. #endif
  1030.         else
  1031. #if HTS_XCONN
  1032.           soc=http_xfopen(test,0,0,NULL,adr,fil,&(back[p].r));
  1033. #else
  1034.         soc=http_xfopen(test,0,1,NULL,adr,fil,&(back[p].r));
  1035. #endif
  1036. #endif
  1037.       } else {
  1038.         soc = back[p].r.soc;
  1039.  
  1040.         if ((opt->debug>1) && (opt->log!=NULL)) {
  1041.           fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): successfully linked #%d (for %s%s)"LF, 
  1042.             back[p].r.debugid,
  1043.             back[p].url_adr, back[p].url_fil); test_flush;
  1044.         }
  1045.       }
  1046.       
  1047.       if (opt->timeout>0) {    // gestion du opt->timeout
  1048.         back[p].timeout=opt->timeout;
  1049.         back[p].timeout_refresh=time_local();
  1050.       } else {
  1051.         back[p].timeout=-1;    // pas de gestion (default)
  1052.       }
  1053.       
  1054.       if (opt->rateout>0) {    // gestion d'un taux minimum de transfert tolΘrΘ
  1055.         back[p].rateout=opt->rateout;
  1056.         back[p].rateout_time=time_local();
  1057.       } else {
  1058.         back[p].rateout=-1;    // pas de gestion (default)
  1059.       }
  1060.  
  1061.       // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas o∙ cela est
  1062.       // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
  1063.       //if ((back[p].statuscode!=200) || (soc<0)) { // ERREUR HTTP/autre
  1064.  
  1065. #if CNXDEBUG
  1066. printf("Xfopen ok, poll..\n");
  1067. #endif
  1068.  
  1069. #if HTS_XGETHOST
  1070.     if (soc!=INVALID_SOCKET)
  1071.       if (back[p].status==101) {  // pas d'erreur
  1072.         if (!back[p].r.is_file)
  1073.           back[p].status=100;   // connexion en cours
  1074.         else
  1075.           back[p].status=1;     // fichier
  1076.       }
  1077.  
  1078. #else
  1079.       if (soc==INVALID_SOCKET) { // erreur socket
  1080.         back[p].status=0;    // FINI
  1081.         //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
  1082.         back[p].r.soc=INVALID_SOCKET;
  1083.       } else {
  1084.         if (!back[p].r.is_file)
  1085. #if HTS_XCONN
  1086.           back[p].status=100;   // connexion en cours
  1087. #else
  1088.           back[p].status=99;    // chargement en tΩte en cours
  1089. #endif
  1090.         else
  1091.           back[p].status=1;     // chargement fichier
  1092. #if BDEBUG==1
  1093.         printf("..loading header\n");
  1094. #endif
  1095.       }
  1096. #endif
  1097.       
  1098.     }
  1099.  
  1100.  
  1101.     // note: si il y a erreur (404,etc) status=2 (terminΘ/Θchec) mais
  1102.     // le lien est considΘrΘ comme traitΘ
  1103.     //if (back[p].soc<0)  // erreur
  1104.     //  return -1;
  1105.  
  1106.     return 0;
  1107.   } else {
  1108.     if ((opt->debug>1) && (opt->errlog!=NULL)) {
  1109.       fspc(opt->errlog,"debug"); fprintf(opt->errlog,"error: no space left in stack for back_add"LF);
  1110.     }
  1111.     return -1;    // plus de place
  1112.   }
  1113. }
  1114.  
  1115.  
  1116.  
  1117. #if HTS_XGETHOST
  1118. #if USE_BEGINTHREAD
  1119. // lancement multithread du robot
  1120. PTHREAD_TYPE Hostlookup(void* iadr_p) {
  1121.   char iadr[256];
  1122.   t_dnscache* cache=_hts_cache();  // adresse du cache
  1123.   t_hostent* hp;
  1124.   int error_found=0;
  1125.  
  1126.   // recopier (aprΦs id:pass)
  1127. #if DEBUGDNS 
  1128.   printf("resolv in background: %s\n",jump_identification(iadr_p));
  1129. #endif
  1130.   strcpybuff(iadr,jump_identification(iadr_p));
  1131.   // couper Θventuel :
  1132.   {
  1133.     char *a;
  1134.     if ( (a=jump_toport(iadr)) )
  1135.       *a='\0';          // get rid of it
  1136.   }
  1137.   freet(iadr_p);
  1138.  
  1139.   // attendre que le cache dns soit prΩt
  1140.   while(_hts_lockdns(-1));  // attendre libΘration
  1141.   _hts_lockdns(1);          // locker
  1142.   while(cache->n) {
  1143.     if (strcmp(cache->iadr,iadr)==0) {
  1144.       error_found=1;
  1145.     }
  1146.     cache=cache->n;    // calculer queue
  1147.   }
  1148.   if (strcmp(cache->iadr,iadr)==0) {
  1149.     error_found=1;
  1150.   }
  1151.  
  1152.   if (!error_found) {
  1153.     // en gros copie de hts_gethostbyname sans le return
  1154.     cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache));
  1155.     if (cache->n!=NULL) {
  1156.       t_fullhostent fullhostent_buffer;
  1157.       strcpybuff(cache->n->iadr,iadr);
  1158.       cache->n->host_length=0;        /* pour le moment rien */
  1159.       cache->n->n=NULL;
  1160.       _hts_lockdns(0);          // dΘlocker
  1161.       
  1162.       /* resolve */
  1163. #if DEBUGDNS 
  1164.       printf("gethostbyname() in progress for %s\n",iadr);
  1165. #endif
  1166.       cache->n->host_length=-1;
  1167.       memset(cache->n->host_addr, 0, sizeof(cache->n->host_addr));
  1168.       hp=vxgethostbyname(iadr, &fullhostent_buffer);
  1169.       if (hp!=NULL) {
  1170.         memcpy(cache->n->host_addr, hp->h_addr, hp->h_length);
  1171.         cache->n->host_length = hp->h_length;
  1172.       }
  1173.     } else 
  1174.     _hts_lockdns(0);          // dΘlocker
  1175.   } else {
  1176. #if DEBUGDNS 
  1177.     printf("aborting resolv for %s (found)\n",iadr);
  1178. #endif
  1179.     _hts_lockdns(0);          // dΘlocker
  1180.   }
  1181.   // fin de copie de hts_gethostbyname
  1182.  
  1183. #if DEBUGDNS 
  1184.   printf("quitting resolv for %s (result: %d)\n",iadr,(cache->n!=NULL)?cache->n->host_length:(-999));
  1185. #endif
  1186.  
  1187.   return PTHREAD_RETURN;     /* _endthread implied  */
  1188. }
  1189. #endif
  1190.  
  1191. // attendre que le host (ou celui du proxy) ait ΘtΘ rΘsolu
  1192. // si c'est un fichier, la rΘsolution est immΘdiate
  1193. // idem pour ftp://
  1194. void back_solve(lien_back* back) {
  1195.   if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) {
  1196.   //## if (back->url_adr[0]!=lOCAL_CHAR) {  // qq chose α prΘparer
  1197.     char* a;
  1198.     if (!(back->r.req.proxy.active))
  1199.       a=back->url_adr;
  1200.     else
  1201.       a=back->r.req.proxy.name;
  1202.     a = jump_protocol(a);
  1203.     if (!hts_dnstest(a)) {   // non encore testΘ!..
  1204.       // inscire en thread
  1205. #if HTS_WIN
  1206.       // Windows
  1207. #if USE_BEGINTHREAD
  1208.       {
  1209.         char* p = calloct(strlen(a)+2,1);
  1210.         if (p) {
  1211.           strcpybuff(p,a);
  1212.           _beginthread( Hostlookup , 0, p );
  1213.         }
  1214.       }
  1215. #else
  1216.       /*t_hostent* h=*/
  1217.       /*hts_gethostbyname(a);*/  // calcul
  1218. #endif
  1219. #else
  1220. #if USE_BEGINTHREAD
  1221.         char* p = calloct(strlen(a)+2,1);
  1222.         if (p) {
  1223.           strcpybuff(p,a);
  1224.           _beginthread( Hostlookup , 0, p );
  1225.         }
  1226. #else
  1227.       // Sous Unix, le gethostbyname() est bloquant..
  1228.       /*t_hostent* h=*/
  1229.       /*hts_gethostbyname(a);*/  // calcul
  1230. #endif
  1231. #endif
  1232.     }
  1233.   }
  1234. }
  1235.  
  1236. // dΘtermine si le host a pu Ωtre rΘsolu
  1237. int host_wait(lien_back* back) {
  1238.   if ((!strfield(back->url_adr,"file://")) && (!strfield(back->url_adr,"ftp://"))) {
  1239.   //## if (back->url_adr[0]!=lOCAL_CHAR) {
  1240.     if (!(back->r.req.proxy.active)) {
  1241.       return (hts_dnstest(back->url_adr));
  1242.     } else {
  1243.       return (hts_dnstest(back->r.req.proxy.name));      
  1244.     }
  1245.   } else return 1;    // prΩt, fichier local
  1246. }
  1247. #endif
  1248.  
  1249.  
  1250. // Θlimine les fichiers non html en backing (anticipation)
  1251. // cleanup non-html files in backing to save backing space
  1252. // and allow faster "save in cache" operation
  1253. // also cleanup keep-alive sockets and ensure that not too many sockets are being opened
  1254. void back_clean(httrackp* opt,cache_back* cache,lien_back* back,int back_max) {
  1255. #if HTS_ANALYSTE
  1256.   int oneMore = ( (_hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (_hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0;  // testing links
  1257. #endif
  1258.   int i;
  1259.   for(i=0;i<back_max;i++) {
  1260.     if (back[i].status == 0) {                                   // ready
  1261.       /* Check autoclean */
  1262.       if (!back[i].testmode) {                                   // not test mode
  1263.         if (strnotempty(back[i].url_sav)) {                      // filename exists
  1264.           if (back[i].r.statuscode==200) {                   // HTTP "OK"
  1265.             if (back[i].r.size>0) {                              // size>0
  1266.               if (back[i].r.is_write                                // not in memory (on disk, ready)
  1267.                 && !is_hypertext_mime(back[i].r.contenttype)        // not HTML/hypertext
  1268.                 && !may_be_hypertext_mime(back[i].r.contenttype)    // may NOT be parseable mime type
  1269.                 ) {
  1270.                 if (back[i].pass2_ptr) {
  1271.                   // finalize
  1272.                   // // back_finalize(opt,cache,back,i);
  1273.                   // stats
  1274.                   //HTS_STAT.stat_bytes+=back[i].r.size;
  1275.                   //HTS_STAT.stat_files++;
  1276.                   //if ( (!back[i].r.notmodified) && (opt->is_update) ) { 
  1277.                   //  HTS_STAT.stat_updated_files++;       // page modifiΘe
  1278.                   //}
  1279.                   //xxxcache_mayadd(opt,cache,&back[i].r,back[i].url_adr,back[i].url_fil,back[i].url_sav);
  1280.                   usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil);
  1281.                   *back[i].pass2_ptr=-1;  // Done!
  1282.                   back_maydelete(opt,back,i);    // May delete backing entry
  1283.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  1284.                     fspc(opt->log,"info"); fprintf(opt->log,"File successfully written in background: %s"LF,back[i].url_sav); test_flush;
  1285.                   }
  1286.                 }
  1287.               } else {
  1288.                 if (!back[i].finalized) {
  1289.                   if (1) {
  1290.                     /* Ensure deleted or recycled socket */
  1291.                     /* BUT DO NOT YET WIPE back[i].r.adr */
  1292.                     back_maydeletehttp(opt, back, back_max, i);
  1293.                     if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1294.                       fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s validated (cached, left in memory)"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1295.                     }
  1296.                   } else {
  1297.                   /*
  1298.                   NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO)
  1299.                     */
  1300.                     /* Lock the entry but do not keep the html data in memory (in cache) */
  1301.                     if (opt->cache) {
  1302.                       htsblk r;
  1303.                       
  1304.                       /* Ensure deleted or recycled socket */
  1305.                       back_maydeletehttp(opt, back, back_max, i);
  1306.                       assertf(back[i].r.soc == INVALID_SOCKET);
  1307.                       
  1308.                       /* Check header */
  1309.                       cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r);
  1310.                       if (r.statuscode == 200) {
  1311.                         if (back[i].r.soc == INVALID_SOCKET) {
  1312.                           /* Delete buffer and sockets */
  1313.                           deleteaddr(&back[i].r);
  1314.                           deletehttp(&back[i].r);
  1315.                           back[i].finalized = 1;
  1316.                           if ( (opt->debug>1) && (opt->log!=NULL) ) {
  1317.                             fspc(opt->log,"debug"); fprintf(opt->log,"file %s%s temporarily left in cache to spare memory"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1318.                           }
  1319.                         }
  1320.                       } else {
  1321.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  1322.                           fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected html cache lookup error during back clean"LF); test_flush;
  1323.                         }            
  1324.                       }
  1325.                       // xxc xxc
  1326.                     }
  1327.                   }
  1328.                 }
  1329.               }
  1330.             }
  1331.           }
  1332.         }
  1333.       }
  1334.     } else if (back[i].status == -103) {                         // waiting (keep-alive)
  1335.       if (
  1336.         ! back[i].r.keep_alive
  1337.         || back[i].r.soc == INVALID_SOCKET
  1338.         || back[i].r.keep_alive_max < 1
  1339.         || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t
  1340.         ) {
  1341.         if ((opt->debug>0) && (opt->log!=NULL)) {
  1342.             fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): live socket closed #%d (%s)"LF, 
  1343.             back[i].r.debugid,
  1344.             back[i].url_adr);
  1345.             test_flush;
  1346.         }
  1347.         back_delete(opt,back, i);    // delete backing entry
  1348.       }
  1349.     }
  1350.   }
  1351.   /* switch connections to live ones */
  1352.   for(i=0;i<back_max;i++) {
  1353.     if (back[i].status == 0) {                                   // ready
  1354.       if (back[i].r.soc != INVALID_SOCKET) {
  1355.         back_maydeletehttp(opt,back, back_max, i);
  1356.       }
  1357.       
  1358.     }
  1359.   }
  1360.   /* delete sockets if too many keep-alive'd sockets in background */
  1361.   if (opt->maxsoc > 0) {
  1362.     int max = opt->maxsoc + oneMore;
  1363.     int curr = back_nsoc_overall(back, back_max);
  1364.     if (curr > max) {
  1365.       if ((opt->debug>1) && (opt->log!=NULL)) {
  1366.         fspc(opt->log,"debug"); fprintf(opt->log,"(Keep-Alive): deleting #%d sockets"LF, 
  1367.           curr - max); test_flush;
  1368.       }
  1369.     }
  1370.     for(i = 0 ; i < back_max && curr > max ; i++) {
  1371.       if (back[i].status == -103) {
  1372.         back_delete(opt,back, i);    // delete backing entry
  1373.         curr--;
  1374.       }
  1375.     }
  1376.   }
  1377. }
  1378.  
  1379.  
  1380. // attente (gestion des buffers des sockets)
  1381. void back_wait(lien_back* back,int back_max,httrackp* opt,cache_back* cache,TStamp stat_timestart) {
  1382.   int i;
  1383.   T_SOC nfds=INVALID_SOCKET;
  1384.   fd_set fds,fds_c,fds_e;     // fds pour lecture, connect (write), et erreur
  1385.   int nsockets;     // nbre sockets
  1386.   LLint max_read_bytes;  // max bytes read per sockets
  1387.   struct timeval tv;
  1388.   int do_wait=0;
  1389.   int gestion_timeout=0;
  1390.   int busy_recv=0;     // pas de donnΘes pour le moment   
  1391.   int busy_state=0;    // pas de connexions
  1392.   int max_loop;  // nombre de boucles max α parcourir..
  1393. #if HTS_ANALYSTE
  1394.   int max_loop_chk=0;
  1395. #endif
  1396.  
  1397.  
  1398.   // max. number of loops
  1399.   max_loop=8;
  1400.  
  1401. #if 1
  1402.   // Cleanup the stack to save space!
  1403.   back_clean(opt,cache,back,back_max);
  1404. #endif
  1405.  
  1406.   // recevoir tant qu'il y a des donnΘes (avec un maximum de max_loop boucles)
  1407.   do_wait=0;
  1408.   gestion_timeout=0;
  1409.   do {
  1410.     int max_c;
  1411.     busy_state=busy_recv=0;
  1412.  
  1413. #if 0
  1414.     check_rate(stat_timestart,opt->maxrate);    // vΘrifier taux de transfert
  1415. #endif
  1416.     // inscrire les sockets actuelles, et rechercher l'ID la plus ΘlevΘe
  1417.     FD_ZERO(&fds);
  1418.     FD_ZERO(&fds_c);
  1419.     FD_ZERO(&fds_e);
  1420.     nsockets=0;
  1421.     max_read_bytes=TAILLE_BUFFER;     // maximum bytes that can be read
  1422.     nfds=INVALID_SOCKET;
  1423.  
  1424.     max_c=1;
  1425.     for(i=0;i<back_max;i++) {
  1426.  
  1427.       // en cas de gestion du connect prΘemptif
  1428. #if HTS_XCONN
  1429.       if (back[i].status==100) {      // connexion
  1430.         do_wait=1;
  1431.  
  1432.         // noter socket write
  1433.         FD_SET(back[i].r.soc,&fds_c);
  1434.         
  1435.         // noter socket erreur
  1436.         FD_SET(back[i].r.soc,&fds_e);
  1437.  
  1438.         // calculer max
  1439.         if (max_c) {
  1440.           max_c=0;
  1441.           nfds=back[i].r.soc;
  1442.         } else if (back[i].r.soc>nfds) {
  1443.           // ID socket la plus ΘlevΘe
  1444.           nfds=back[i].r.soc;
  1445.         }
  1446.         
  1447.       } else
  1448. #endif
  1449. #if HTS_XGETHOST
  1450.       if (back[i].status==101) {      // attente
  1451.         // rien α faire..
  1452.       } else
  1453. #endif
  1454.       // poll pour la lecture sur les sockets
  1455.       if ((back[i].status>0) && (back[i].status<100)) {  // en rΘception http
  1456.             
  1457. #if BDEBUG==1
  1458.         //printf("....socket in progress: %d\n",back[i].r.soc);
  1459. #endif
  1460.         // non local et non ftp
  1461.         if (!back[i].r.is_file) {
  1462.         //## if (back[i].url_adr[0]!=lOCAL_CHAR) {
  1463.           
  1464.           // vΘrification de sΘcuritΘ
  1465.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  1466.             do_wait=1;
  1467.             
  1468.             // noter socket read
  1469.             FD_SET(back[i].r.soc,&fds);
  1470.             
  1471.             // noter socket error
  1472.             FD_SET(back[i].r.soc,&fds_e);
  1473.             
  1474.             // incrΘmenter nombre de sockets
  1475.             nsockets++;
  1476.  
  1477.             // calculer max
  1478.             if (max_c) {
  1479.               max_c=0;
  1480.               nfds=back[i].r.soc;
  1481.             } else if (back[i].r.soc>nfds) {
  1482.               // ID socket la plus ΘlevΘe
  1483.               nfds=back[i].r.soc;
  1484.             }
  1485.           } else {
  1486.             back[i].r.statuscode=-4;
  1487.             if (back[i].status==100)
  1488.               strcpybuff(back[i].r.msg,"Connect Error");
  1489.             else
  1490.               strcpybuff(back[i].r.msg,"Receive Error");
  1491.             back[i].status=0;  // terminΘ
  1492.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1493.               fspc(opt->log,"warning"); fprintf(opt->log,"Unexpected socket error during pre-loop"LF); test_flush;
  1494.             }            
  1495.           }
  1496. #if WIDE_DEBUG
  1497.           else {
  1498.             DEBUG_W("PANIC!!! Socket is invalid in a poll test!\n");
  1499.           }
  1500. #endif
  1501.           
  1502.         }
  1503.         
  1504.       }
  1505.     }    
  1506.     nfds++;
  1507.     
  1508.     if (do_wait) {  // attendre
  1509.       // temps d'attente max: 2.5 seconde
  1510.       tv.tv_sec=HTS_SOCK_SEC;
  1511.       tv.tv_usec=HTS_SOCK_MS;
  1512.       
  1513. #if BDEBUG==1
  1514.       printf("..select\n");
  1515. #endif
  1516.       
  1517.       // poller les sockets-attention au noyau sous Unix..
  1518. #if HTS_WIDE_DEBUG    
  1519.       DEBUG_W("select\n");
  1520. #endif
  1521.       select(nfds,&fds,&fds_c,&fds_e,&tv);
  1522. #if HTS_WIDE_DEBUG    
  1523.       DEBUG_W("select done\n");
  1524. #endif      
  1525.     }
  1526.     
  1527.     // maximum data which can be received for a socket, if limited
  1528.     if (nsockets) {
  1529.       if (opt->maxrate>0) {
  1530.         max_read_bytes = ( check_downloadable_bytes(opt->maxrate) / nsockets );
  1531.         if (max_read_bytes > TAILLE_BUFFER) {
  1532.           /* limit size */
  1533.           max_read_bytes = TAILLE_BUFFER;
  1534.         } else if (max_read_bytes < TAILLE_BUFFER) {
  1535.           /* a small pause */
  1536.           Sleep(10);
  1537.         }
  1538.       }
  1539.     }
  1540.     if (!max_read_bytes)
  1541.       busy_recv=0;
  1542.     
  1543.     // recevoir les donnΘes arrivΘes
  1544.     for(i=0;i<back_max;i++) {
  1545.       
  1546.       if (back[i].status>0) {
  1547.         if (!back[i].r.is_file) {  // not file..
  1548.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  1549.             int err=FD_ISSET(back[i].r.soc,&fds_e);
  1550.             if (err) {
  1551.               if (back[i].r.soc!=INVALID_SOCKET) {
  1552. #if HTS_DEBUG_CLOSESOCK
  1553.                 DEBUG_W("back_wait: deletehttp\n");
  1554. #endif
  1555.                 deletehttp(&back[i].r);
  1556.               }
  1557.               back[i].r.soc=INVALID_SOCKET;
  1558.               back[i].r.statuscode=-4;
  1559.               if (back[i].status==100)
  1560.                 strcpybuff(back[i].r.msg,"Connect Error");
  1561.               else
  1562.                 strcpybuff(back[i].r.msg,"Receive Error");
  1563.               if (back[i].status == -103) {     /* Keep-alive socket */
  1564.                 back_delete(opt,back, i);
  1565.               } else {
  1566.                 back[i].status=0;  // terminΘ
  1567.               }
  1568.             }
  1569.           }
  1570.         }
  1571.       }
  1572.       
  1573.       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
  1574.       if (back[i].status==100) {   // attendre connect
  1575.         int dispo=0;
  1576.         // vΘrifier l'existance de timeout-check
  1577.         if (!gestion_timeout)
  1578.           if (back[i].timeout>0)
  1579.             gestion_timeout=1;
  1580.           
  1581.           // connectΘ?
  1582.           dispo=FD_ISSET(back[i].r.soc,&fds_c);
  1583.           if (dispo) {    // ok connected!!
  1584.             busy_state=1;
  1585.             
  1586. #if HTS_USEOPENSSL
  1587.             /* SSL mode */
  1588.             if (SSL_is_available && back[i].r.ssl) {
  1589.               // handshake not yet launched
  1590.               if (!back[i].r.ssl_con) {
  1591.                 SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
  1592.                 // new session
  1593.                 back[i].r.ssl_con = SSL_new(openssl_ctx);
  1594.                 if (back[i].r.ssl_con) {
  1595.                   SSL_clear(back[i].r.ssl_con);
  1596.                   if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) {
  1597.                     SSL_set_connect_state(back[i].r.ssl_con);
  1598.                     back[i].status = 102;         /* handshake wait */
  1599.                   } else
  1600.                     back[i].r.statuscode=-6;
  1601.                 } else
  1602.                   back[i].r.statuscode=-6;
  1603.               }
  1604.               /* Error */
  1605.               if (back[i].r.statuscode == -6) {
  1606.                 strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
  1607.                 deletehttp(&back[i].r);
  1608.                 back[i].r.soc=INVALID_SOCKET;
  1609.                 back[i].r.statuscode=-5;
  1610.                 back[i].status=0;
  1611.               }
  1612.             }
  1613.             
  1614. #endif
  1615.  
  1616. #if BDEBUG==1
  1617.           printf("..connect ok on socket %d\n",back[i].r.soc);
  1618. #endif
  1619.           
  1620.           if ((back[i].r.soc != INVALID_SOCKET) && (back[i].status==100)) {
  1621.             /* limit nb. connections/seconds to avoid server overload */
  1622.             if (opt->maxconn>0) {
  1623.               Sleep(1000/opt->maxconn);
  1624.             }
  1625.             
  1626.             back[i].ka_time_start=time_local();
  1627.             if (back[i].timeout>0) {    // refresh timeout si besoin est
  1628.               back[i].timeout_refresh=back[i].ka_time_start;
  1629.             }
  1630.             if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  1631.               back[i].rateout_time=back[i].ka_time_start;
  1632.             }
  1633.             // envoyer header
  1634.             //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
  1635.             HTS_STAT.stat_nrequests++;
  1636.             if (!back[i].head_request)
  1637.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1638.             else if (back[i].head_request==2)  // test en GET!
  1639.               http_sendhead(opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1640.             else        // test!
  1641.               http_sendhead(opt->cookie,1,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  1642.             back[i].status=99;  // attendre en tΩte maintenant
  1643.           }
  1644.         }
  1645.         
  1646.         // attente gethostbyname
  1647.       }
  1648. #if HTS_USEOPENSSL
  1649.       else if (SSL_is_available && back[i].status==102) {   // wait for SSL handshake
  1650.         /* SSL mode */
  1651.         if (back[i].r.ssl) {
  1652.           int conn_code;
  1653.           if ((conn_code = SSL_connect(back[i].r.ssl_con)) <= 0) {
  1654.             /* non blocking I/O, will retry */
  1655.             int err_code = SSL_get_error(back[i].r.ssl_con, conn_code);
  1656.             if (
  1657.               (err_code != SSL_ERROR_WANT_READ)
  1658.               &&
  1659.               (err_code != SSL_ERROR_WANT_WRITE)
  1660.               ) {
  1661.               char tmp[256];
  1662.               tmp[0]='\0';
  1663.               ERR_error_string(err_code, tmp);
  1664.               back[i].r.msg[0]='\0';
  1665.               strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2);
  1666.               if (!strnotempty(back[i].r.msg)) {
  1667.                 sprintf(back[i].r.msg, "SSL/TLS error %d", err_code);
  1668.               }
  1669.               deletehttp(&back[i].r);
  1670.               back[i].r.soc=INVALID_SOCKET;
  1671.               back[i].r.statuscode=-5;
  1672.               back[i].status=0;
  1673.             }
  1674.           } else {        /* got it! */
  1675.             back[i].status=100;       // back to waitconnect
  1676.           }
  1677.         } else {
  1678.           strcpybuff(back[i].r.msg, "unexpected SSL/TLS error");
  1679.           deletehttp(&back[i].r);
  1680.           back[i].r.soc=INVALID_SOCKET;
  1681.           back[i].r.statuscode=-5;
  1682.           back[i].status=0;
  1683.         }
  1684.         
  1685.       }
  1686. #endif
  1687. #if HTS_XGETHOST
  1688.       else if (back[i].status==101) {  // attendre gethostbyname
  1689. #if DEBUGDNS 
  1690.         //printf("status 101 for %s\n",back[i].url_adr);
  1691. #endif
  1692.  
  1693.         if (!gestion_timeout)
  1694.           if (back[i].timeout>0)
  1695.             gestion_timeout=1;
  1696.  
  1697.         if (host_wait(&back[i])) {    // prΩt
  1698.           back[i].status=100;        // attente connexion
  1699.           if (back[i].timeout>0) {    // refresh timeout si besoin est
  1700.             back[i].timeout_refresh=time_local();
  1701.           }
  1702.           if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  1703.             back[i].rateout_time=time_local();
  1704.           }
  1705.  
  1706.           back[i].r.soc=http_xfopen(0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r));
  1707.           if (back[i].r.soc==INVALID_SOCKET) {
  1708.             back[i].status=0;  // fini, erreur
  1709.             if (back[i].r.soc!=INVALID_SOCKET) {
  1710. #if HTS_DEBUG_CLOSESOCK
  1711.               DEBUG_W("back_wait(2): deletehttp\n");
  1712. #endif
  1713.               deletehttp(&back[i].r);
  1714.             }
  1715.             back[i].r.soc=INVALID_SOCKET;
  1716.             back[i].r.statuscode=-5;
  1717.             if (strnotempty(back[i].r.msg)==0) 
  1718.               strcpybuff(back[i].r.msg,"Unable to resolve host name");
  1719.           }
  1720.         }
  1721.         
  1722.  
  1723.       // ---- FLAG READ MIS A UN?: POUR LA RECEPTION
  1724.       }
  1725. #endif
  1726. #if USE_BEGINTHREAD
  1727.       // ..rien α faire, c'est magic les threads
  1728. #else
  1729.       else if (back[i].status==1000) {  // en rΘception ftp
  1730.         if (!fexist(back[i].location_buffer)) {    // terminΘ
  1731.           FILE* fp;
  1732.           fp=fopen(fconcat(back[i].location_buffer,".ok"),"rb");
  1733.           if (fp) {
  1734.             int j=0;
  1735.             fscanf(fp,"%d ",&(back[i].r.statuscode));
  1736.             while(!feof(fp)) {
  1737.               int c = fgetc(fp);
  1738.               if (c!=EOF)
  1739.                 back[i].r.msg[j++]=c;
  1740.             }
  1741.             back[i].r.msg[j++]='\0';
  1742.             fclose(fp);
  1743.             remove(fconcat(back[i].location_buffer,".ok"));
  1744.             strcpybuff(fconcat(back[i].location_buffer,".ok"),"");
  1745.           } else {
  1746.             strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok");
  1747.             back[i].r.statuscode=-1;
  1748.           }
  1749.           back[i].status=0;
  1750.           // finalize transfer
  1751.           if (back[i].r.statuscode>0) {
  1752.             back_finalize(opt,cache,back,i);
  1753.           }
  1754.         }
  1755.       }
  1756. #endif
  1757.       else if ((back[i].status>0) && (back[i].status<1000)) {  // en rΘception http
  1758.         int dispo=0;
  1759.         
  1760.         // vΘrifier l'existance de timeout-check
  1761.         if (!gestion_timeout)
  1762.           if (back[i].timeout>0)
  1763.             gestion_timeout=1;
  1764.           
  1765.           // donnΘes dispo?
  1766.           //## if (back[i].url_adr[0]!=lOCAL_CHAR)
  1767.           if (!back[i].r.is_file) {
  1768.             dispo=FD_ISSET(back[i].r.soc,&fds);
  1769.           }
  1770.           else
  1771.             dispo=1;
  1772.  
  1773.           // Check transfer rate!
  1774.           if (!max_read_bytes)
  1775.             dispo=0;                // limit transfer rate
  1776.           
  1777.           if (dispo) {    // donnΘes dispo
  1778.             LLint retour_fread;
  1779.             busy_recv=1;    // on rΘcupΦre encore
  1780. #if BDEBUG==1
  1781.             printf("..data available on socket %d\n",back[i].r.soc);
  1782. #endif
  1783.  
  1784.             
  1785.             // range size hack old location
  1786.  
  1787. #if HTS_DIRECTDISK
  1788.             // Court-circuit:
  1789.             // Peut-on stocker le fichier directement sur disque?
  1790.             // Ahh que ca serait vachement mieux et que ahh que la mΘmoire vous dit merci!
  1791.             if (back[i].status) {
  1792.               if (back[i].r.is_write==0) {  // mode mΘmoire
  1793.                 if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  1794.                   if (!back[i].testmode) {  // pas mode test
  1795.                     if (strnotempty(back[i].url_sav)) {
  1796.                       if (strcmp(back[i].url_fil,"/robots.txt")) {
  1797.                         if (back[i].r.statuscode==200) {  // 'OK'
  1798.                           if (!is_hypertext_mime(back[i].r.contenttype)) {    // pas HTML
  1799.                             if (opt->getmode&2) {    // on peut ecrire des non html
  1800.                               int fcheck=0;
  1801.                               back[i].r.is_write=1;    // Θcrire
  1802.                               if (back[i].r.compressed
  1803.                                 &&
  1804.                                 /* .gz are *NOT* depacked!! */
  1805.                                 (strfield(get_ext(back[i].url_sav),"gz") == 0)
  1806.                                 ) {
  1807.                                 back[i].tmpfile_buffer[0]='\0';
  1808.                                 back[i].tmpfile=tmpnam(back[i].tmpfile_buffer);
  1809.                                 if (back[i].tmpfile != NULL && back[i].tmpfile[0])
  1810.                                   back[i].r.out=fopen(back[i].tmpfile,"wb");
  1811.                               } else {
  1812.                                 back[i].r.compressed=0;
  1813.                                 back[i].r.out=filecreate(back[i].url_sav);
  1814.                               }
  1815.                               if (back[i].r.out==NULL) {
  1816.                                 if ((fcheck=check_fatal_io_errno())) {
  1817.                                   opt->state.exit_xh=-1;   /* fatal error */
  1818.                                 }
  1819.                               }
  1820. #if HDEBUG
  1821.                               printf("direct-disk: %s\n",back[i].url_sav);
  1822. #endif
  1823.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  1824.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File received from net to disk: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1825.                               }
  1826.                               
  1827.                               if (back[i].r.out==NULL) {
  1828.                                 if (opt->errlog) {
  1829.                                   fspc(opt->errlog,"error");
  1830.                                   fprintf(opt->errlog,"Unable to save file %s : %s"LF,back[i].url_sav, strerror(errno));
  1831.                                   if (fcheck) {
  1832.                                     fspc(opt->errlog,"error");
  1833.                                     fprintf(opt->errlog,"* * Fatal write error, giving up"LF);
  1834.                                   }
  1835.                                   test_flush;
  1836.                                 }
  1837.                                 back[i].r.is_write=0;    // erreur, abandonner
  1838. #if HDEBUG
  1839.                                 printf("..error!\n");
  1840. #endif
  1841.                               }
  1842. #if HTS_WIN==0
  1843.                               else chmod(back[i].url_sav,HTS_ACCESS_FILE);      
  1844. #endif          
  1845.                             } else {  // on coupe tout!
  1846.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  1847.                                 fspc(opt->log,"debug"); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  1848.                               }
  1849.                               back[i].status=0;  // terminΘ
  1850.                               if (!back[i].testmode)
  1851.                                 back[i].r.statuscode=-10;    // EUHH CANCEL
  1852.                               else
  1853.                                 back[i].r.statuscode=-10;    // "TEST OK"
  1854.                               if (back[i].r.soc!=INVALID_SOCKET) {
  1855. #if HTS_DEBUG_CLOSESOCK
  1856.                                 DEBUG_W("back_wait(3): deletehttp\n");
  1857. #endif
  1858.                                 deletehttp(&back[i].r);
  1859.                               }
  1860.                               back[i].r.soc=INVALID_SOCKET;
  1861.                             }
  1862.                           }
  1863.                         }
  1864.                       }
  1865.                     }
  1866.                   }
  1867.                 }
  1868.               }
  1869.             }
  1870. #endif              
  1871.  
  1872.             // rΘception de donnΘes depuis socket ou fichier
  1873.             if (back[i].status) {
  1874.               if (back[i].status==99)  // recevoir par bloc de lignes
  1875.                 retour_fread=http_xfread1(&(back[i].r),0);
  1876.               else if (back[i].status==98 || back[i].status==97) { // recevoir longueur chunk en hexa caractΦre par caractΦre
  1877.                 // backuper pour lire dans le buffer chunk
  1878.                 htsblk r;
  1879.                 memcpy(&r, &(back[i].r), sizeof(htsblk));
  1880.                 back[i].r.is_write=0;                   // mΘmoire
  1881.                 back[i].r.adr=back[i].chunk_adr;        // adresse
  1882.                 back[i].r.size=back[i].chunk_size;      // taille taille chunk
  1883.                 back[i].r.totalsize=-1;                 // total inconnu
  1884.                 back[i].r.out=NULL;
  1885.                 back[i].r.is_file=0;
  1886.                 //
  1887.                 // ligne par ligne
  1888.                 retour_fread=http_xfread1(&(back[i].r),-1);
  1889.                 // modifier et restaurer
  1890.                 back[i].chunk_adr=back[i].r.adr;        // adresse
  1891.                 back[i].chunk_size=back[i].r.size;      // taille taille chunk
  1892.                 memcpy(&(back[i].r), &r, sizeof(htsblk));    // restaurer vΘritable r
  1893.               }
  1894.               else if (back[i].is_chunk) {         // attention chunk, limiter taille α lire
  1895. #if CHUNKDEBUG==1
  1896.                 printf("[%d] read %d bytes\n",(int)back[i].r.soc,(int)min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  1897. #endif
  1898.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  1899.               } else              
  1900.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) max_read_bytes);
  1901.                 // retour_fread=http_fread1(&(back[i].r));
  1902.             } else
  1903.               retour_fread=-1;                    // interruption ou annulation interne (peut ne pas Ωtre une erreur)
  1904.             
  1905.             // Si rΘception chunk, tester si on est pas α la fin!
  1906.             if (back[i].status==1) {
  1907.               if (back[i].is_chunk) {     // attendre prochain chunk
  1908.                 if (back[i].r.size==back[i].r.totalsize) {      // fin chunk!
  1909.                   //printf("chunk end at %d\n",back[i].r.size);
  1910.                   back[i].status=97;  /* fetch ending CRLF */
  1911.                   if (back[i].chunk_adr!=NULL) { 
  1912.                     freet(back[i].chunk_adr); 
  1913.                     back[i].chunk_adr=NULL; 
  1914.                   } 
  1915.                   back[i].chunk_size=0;
  1916.                   retour_fread=0;       // pas d'erreur
  1917. #if CHUNKDEBUG==1
  1918.                   printf("[%d] waiting for current chunk CRLF..\n",(int)back[i].r.soc);
  1919. #endif
  1920.                 }
  1921.               } else if (back[i].r.keep_alive) {
  1922.                 if (back[i].r.size==back[i].r.totalsize) {      // fin!
  1923.                   retour_fread=-1;       // end
  1924.                 }
  1925.               }
  1926.             }
  1927.             
  1928.             if (retour_fread < 0) {    // fin rΘception
  1929.               back[i].status=0;    // terminΘ
  1930.               if (back[i].r.soc!=INVALID_SOCKET) {
  1931. #if HTS_DEBUG_CLOSESOCK
  1932.                 DEBUG_W("back_wait(4): deletehttp\n");
  1933. #endif
  1934.                 /*KA deletehttp(&back[i].r);*/
  1935.                 back_maydeletehttp(opt, back, back_max, i);
  1936.               }
  1937.               /*KA back[i].r.soc=INVALID_SOCKET; */
  1938. #if CHUNKDEBUG==1
  1939.               if (back[i].is_chunk)
  1940.                 printf("[%d] must be the last chunk for %s (connection closed) - %d/%d\n",(int)back[i].r.soc,back[i].url_fil,back[i].r.size,back[i].r.totalsize);
  1941. #endif
  1942.               //if ((back[i].r.statuscode==-1) && (strnotempty(back[i].r.msg)==0)) {
  1943.               if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) {
  1944. #if HDEBUG
  1945.                 printf("error interruped: %s\n",back[i].r.adr);
  1946. #endif        
  1947.                 if (back[i].r.size>0)
  1948.                   strcatbuff(back[i].r.msg,"Interrupted transfer");
  1949.                 else
  1950.                   strcatbuff(back[i].r.msg,"No data (connection closed)");
  1951.                 back[i].r.statuscode=-4;
  1952.               }
  1953.  
  1954.               // finalize transfer
  1955.               if (back[i].r.statuscode>0) {
  1956.                 back_finalize(opt,cache,back,i);
  1957.               }
  1958.  
  1959.               if (back[i].r.totalsize>0) {    // tester totalsize
  1960.               //if ((back[i].r.totalsize>0) && (back[i].status==99)) {    // tester totalsize
  1961.                 if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  1962.                   if (!opt->tolerant) {
  1963.                     //#if HTS_CL_IS_FATAL
  1964.                     deleteaddr(&back[i].r);
  1965.                     if (back[i].r.size<back[i].r.totalsize)
  1966.                       back[i].r.statuscode=-4;        // recatch
  1967.                     sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize);
  1968.                   } else {
  1969.                     //#else
  1970.                     // Un warning suffira..
  1971.                     if (cache->errlog!=NULL) {
  1972.                       fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  1973.                     }
  1974.                     //#endif
  1975.                   }
  1976.                 }
  1977.               }
  1978. #if BDEBUG==1
  1979.               printf("transfer ok\n");
  1980. #endif
  1981.             } else if (retour_fread > 0) {    // pas d'erreur de rΘception et data
  1982.               if (back[i].timeout>0) {    // refresh timeout si besoin est
  1983.                 back[i].timeout_refresh=time_local();
  1984.               }
  1985.  
  1986.               // Traitement des en tΩtes chunks ou en tΩtes
  1987.               if (back[i].status==98 || back[i].status==97) {        // rΘception taille chunk en hexa (  aprΦs les en tΩtes, peut ne pas
  1988.                 if (back[i].chunk_size > 0 && back[i].chunk_adr[back[i].chunk_size-1]==10) {
  1989.                   int chunk_size=-1;
  1990.                   char chunk_data[64];
  1991.                   if (back[i].chunk_size<32) {      // pas trop gros
  1992.                     char* chstrip=back[i].chunk_adr;
  1993.                     back[i].chunk_adr[ back[i].chunk_size-1]='\0';    // octet nul 
  1994.                     // skip leading spaces or cr
  1995.                     while(isspace(*chstrip)) chstrip++;
  1996.                     chunk_data[0] = '\0';
  1997.                     strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2);
  1998.                     // strip chunk-extension
  1999.                     while( (chstrip = strchr(chunk_data, ';'))) *chstrip='\0';
  2000.                     while( (chstrip = strchr(chunk_data, ' '))) *chstrip='\0';
  2001.                     while( (chstrip = strchr(chunk_data, '\r'))) *chstrip='\0';
  2002. #if CHUNKDEBUG==1
  2003.                     printf("[%d] chunk received and read: %s\n",(int)back[i].r.soc,chunk_data);
  2004. #endif
  2005.                     if (back[i].r.totalsize<0)
  2006.                       back[i].r.totalsize=0;        // initialiser α 0
  2007.                     if (back[i].status==98) {   // "real" chunk
  2008.                       if (sscanf(chunk_data,"%x",&chunk_size) == 1) {
  2009.                         if (chunk_size > 0)
  2010.                           back[i].chunk_blocksize = chunk_size;  /* the data block chunk size */
  2011.                         else
  2012.                           back[i].chunk_blocksize = -1;  /* ending */
  2013.                         back[i].r.totalsize+=chunk_size;    // noter taille
  2014.                         back[i].r.adr=(char*) realloct(back[i].r.adr,(INTsys) back[i].r.totalsize + 1);
  2015.                         if (!back[i].r.adr) {
  2016.                           if (cache->errlog!=NULL) {
  2017.                             fprintf(cache->errlog,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2018.                           }
  2019.                         }
  2020. #if CHUNKDEBUG==1
  2021.                         printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize);
  2022. #endif
  2023.                       } else {
  2024.                         if (cache->errlog!=NULL) {
  2025.                           fprintf(cache->errlog,"Warning: Illegal chunk (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2026.                         }
  2027.                       }
  2028.                     } else {   /* back[i].status==97 : just receiving ending CRLF after data */
  2029.                       if (chunk_data[0] == '\0') {
  2030.                         if (back[i].chunk_blocksize > 0)
  2031.                           chunk_size=(int)back[i].chunk_blocksize;  /* recent data chunk size */
  2032.                         else if (back[i].chunk_blocksize == -1)
  2033.                           chunk_size=0;                        /* ending chunk */
  2034.                         else
  2035.                           chunk_size=1;                        /* fake positive size for 1st chunk history */
  2036. #if CHUNKDEBUG==1
  2037.                         printf("[%d] chunk CRLF seen\n", (int)back[i].r.soc);
  2038. #endif
  2039.                       } else {
  2040.                         if (cache->errlog!=NULL) {
  2041.                           fprintf(cache->errlog,"Warning: Illegal chunk CRLF (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2042.                         }
  2043. #if CHUNKDEBUG==1
  2044.                         printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int)back[i].r.soc, chunk_data);
  2045. #endif
  2046.                       }
  2047.                     }
  2048.                   } else {                                  
  2049.                     if (cache->errlog!=NULL) {
  2050.                       fprintf(cache->errlog,"Warning: Chunk too big ("LLintP") for %s%s"LF,(LLint)back[i].chunk_size,back[i].url_adr,back[i].url_fil);
  2051.                     }
  2052.                   }
  2053.                     
  2054.                   // ok, continuer sur le body
  2055.                     
  2056.                   // si chunk non nul continuer (ou commencer)
  2057.                   if (back[i].status==97 && chunk_size > 0) {
  2058.                     back[i].status = 98;  /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */
  2059. #if CHUNKDEBUG==1
  2060.                     printf("[%d] waiting for next chunk\n", (int)back[i].r.soc);
  2061. #endif
  2062.                   } else if (back[i].status==98 && chunk_size == 0) {  /* final chunk */
  2063.                     back[i].status=97;  /* final CRLF */
  2064. #if CHUNKDEBUG==1
  2065.                     printf("[%d] waiting for final CRLF (chunk)\n", (int)back[i].r.soc);
  2066. #endif
  2067.                   } else if (back[i].status==98 && chunk_size >= 0) {  /* will fetch data now */
  2068.                     back[i].status=1;     // continuer body    
  2069. #if CHUNKDEBUG==1
  2070.                     printf("[%d] waiting for body (chunk)\n", (int)back[i].r.soc);
  2071. #endif
  2072.                   } else {                /* zero-size-chunk-CRLF (end) or error */
  2073. #if CHUNKDEBUG==1
  2074.                     printf("[%d] chunk end, total: %d\n",(int)back[i].r.soc,back[i].r.size);
  2075. #endif
  2076.                     /* End */
  2077.                     //if (back[i].status==97) {
  2078.                     back[i].status=0;     // fin  
  2079.                     //}
  2080.  
  2081.                     // finalize transfer
  2082.                     back_finalize(opt,cache,back,i);
  2083.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2084. #if HTS_DEBUG_CLOSESOCK
  2085.                       DEBUG_W("back_wait(5): deletehttp\n");
  2086. #endif
  2087.                       /* Error */
  2088.                       if (chunk_size < 0) {
  2089.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2090.                         deleteaddr(&back[i].r);
  2091.                         back[i].r.statuscode=-1;
  2092.                         strcpybuff(back[i].r.msg,"Invalid chunk");
  2093. #if CHUNKDEBUG==1
  2094.                         printf("[%d] chunk error\n", (int)back[i].r.soc);
  2095. #endif
  2096.                       } else /* if chunk_size == 0 */ {
  2097. #if CHUNKDEBUG==1
  2098.                         printf("[%d] all chunks now received\n", (int)back[i].r.soc);
  2099. #endif
  2100.                           
  2101.                         /* Tester totalsize en fin de chunk */
  2102.                         if ((back[i].r.totalsize>0)) {    // tester totalsize
  2103.                           if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2104. #if HTS_CL_IS_FATAL
  2105.                             deleteaddr(&back[i].r);
  2106.                             back[i].r.statuscode=-1;
  2107.                             strcpybuff(back[i].r.msg,"Incorrect length");
  2108. #else
  2109.                             // Un warning suffira..
  2110.                             if (cache->errlog!=NULL) {
  2111.                               fspc(cache->errlog,"warning"); fprintf(cache->errlog,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2112.                             }
  2113. #endif
  2114.                           }
  2115.                         }
  2116.                           
  2117.                         /* Oops, trailers! */
  2118.                         if (back[i].r.keep_alive_trailers) {
  2119.                           /* fixme (not yet supported) */
  2120.                         }
  2121.                           
  2122.                       }
  2123.                         
  2124.                         
  2125.                     }
  2126.                   }
  2127.                     
  2128.                   // effacer buffer (chunk en tete)
  2129.                   if (back[i].chunk_adr!=NULL) {
  2130.                     freet(back[i].chunk_adr);
  2131.                     back[i].chunk_adr=NULL;
  2132.                     back[i].chunk_size=0;
  2133.                     // NO! xxback[i].chunk_blocksize = 0;
  2134.                   }
  2135.                   
  2136.                 } // taille buffer chunk > 1 && LF
  2137.                 //
  2138.               } else if (back[i].status==99) {        // en tΩtes (avant le chunk si il est prΘsent)
  2139.                 //
  2140.                 if (back[i].r.size>=2) {
  2141.                   // double LF
  2142.                   if (
  2143.                     ((back[i].r.adr[back[i].r.size-1]==10) && (back[i].r.adr[back[i].r.size-2]==10)) 
  2144.                     ||
  2145.                     (back[i].r.adr[0] == '<')    /* bogus server */
  2146.                     ) {
  2147.                     char rcvd[2048];
  2148.                     int ptr=0;
  2149.                     int noFreebuff=0;
  2150.                     
  2151. #if BDEBUG==1
  2152.                     printf("..ok, header received\n");
  2153. #endif
  2154.                     
  2155.                     
  2156.                     // Callback
  2157. #if HTS_ANALYSTE
  2158.                     if (hts_htmlcheck_receivehead != NULL) {
  2159.                       int test_head=hts_htmlcheck_receivehead(back[i].r.adr, back[i].url_adr, back[i].url_fil, back[i].referer_adr, back[i].referer_fil, &back[i].r);
  2160.                       if (test_head!=1) {
  2161.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  2162.                           fspc(opt->log,"warning"); fprintf(opt->log,"External wrapper aborted transfer, breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2163.                         }
  2164.                         back[i].status=0;  // FINI
  2165.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2166.                         strcpybuff(back[i].r.msg,"External wrapper aborted transfer");
  2167.                         back[i].r.statuscode = -1;
  2168.                       }
  2169.                     }
  2170. #endif
  2171.                     
  2172.                     /* Hack for zero-length headers */
  2173.                     if (back[i].status != 0 && back[i].r.adr[0] != '<') {
  2174.                       
  2175.                       // ----------------------------------------
  2176.                       // traiter en-tΩte!
  2177.                       // status-line α rΘcupΘrer
  2178.                       ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2179.                       if (strnotempty(rcvd)==0) {
  2180.                         /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */
  2181.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  2182.                       }
  2183.                       
  2184.                       // traiter status-line
  2185.                       treatfirstline(&back[i].r,rcvd);
  2186.                       
  2187. #if HDEBUG
  2188.                       printf("(Buffer) Status-Code=%d\n",back[i].r.statuscode);
  2189. #endif
  2190.                       if (_DEBUG_HEAD) {
  2191.                         if (ioinfo) {
  2192.                           fprintf(ioinfo,"[%d] response for %s%s:\r\ncode=%d\r\n",
  2193.                             back[i].r.debugid, jump_identification(back[i].url_adr),back[i].url_fil,back[i].r.statuscode);
  2194.                           fprintfio(ioinfo,back[i].r.adr,">>> ");
  2195.                           fprintf(ioinfo,"\r\n");
  2196.                           fflush(ioinfo);
  2197.                         }                    // en-tΩte
  2198.                       }
  2199.                       
  2200.                       // header // ** !attention! HTTP/0.9 non supportΘ
  2201.                       do {
  2202.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);          
  2203. #if HDEBUG
  2204.                         printf("(buffer)>%s\n",rcvd);      
  2205. #endif
  2206.                         /*
  2207.                         if (_DEBUG_HEAD) {
  2208.                         if (ioinfo) {
  2209.                         fprintf(ioinfo,"(buffer)>%s\r\n",rcvd);      
  2210.                         fflush(ioinfo);
  2211.                         }
  2212.                         }
  2213.                         */
  2214.                         
  2215.                         if (strnotempty(rcvd))
  2216.                           treathead(opt->cookie,back[i].url_adr,back[i].url_fil,&back[i].r,rcvd);  // traiter
  2217.                         
  2218.                         // parfois les serveurs buggΘs renvoient un content-range avec un 200
  2219.                         if (back[i].r.statuscode==200)  // 'OK'
  2220.                           if (strfield(rcvd,"content-range:"))  // Avec un content-range: relisez les RFC..
  2221.                             back[i].r.statuscode=206;    // FORCER A 206 !!!!!
  2222.                           
  2223.                       } while(strnotempty(rcvd));
  2224.                       // ----------------------------------------                    
  2225.  
  2226.                       // libΘrer mΘmoire  -- aprΦs! --
  2227.                       deleteaddr(&back[i].r);
  2228.                     } else {
  2229.                       // assume text/html, OK
  2230.                       treatfirstline(&back[i].r, back[i].r.adr);
  2231.                       noFreebuff=1;
  2232.                     }
  2233.                       
  2234.                   
  2235.                     
  2236.                     /* 
  2237.                     Status code and header-response hacks
  2238.                     */
  2239.  
  2240.                     
  2241.                     // Check response : 203 == 200
  2242.                     if (back[i].r.statuscode==203) {  // 'Non-Authoritative Information'
  2243.                       back[i].r.statuscode=200;       // forcer "OK"
  2244.                     } else if (back[i].r.statuscode == 100) {
  2245.                       back[i].status=99;
  2246.                       back[i].r.size=0;
  2247.                       back[i].r.totalsize=0;
  2248.                       back[i].chunk_size=0;
  2249.                       back[i].r.statuscode=-1;
  2250.                       back[i].r.msg[0]='\0';
  2251.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2252.                         fspc(opt->log,"debug"); fprintf(opt->log,"Status 100 detected for %s%s, continuing headers"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2253.                       }
  2254.                       continue;
  2255.                     }
  2256.                     
  2257.                     /*
  2258.                     Solve "false" 416 problems
  2259.                     */
  2260.                     if (back[i].r.statuscode==416) {  // 'Requested Range Not Satisfiable'
  2261.                       // Example:
  2262.                       // Range: bytes=2830-
  2263.                       // ->
  2264.                       // Content-Range: bytes */2830
  2265.                       if (back[i].range_req_size == back[i].r.crange) {
  2266.                         filenote(back[i].url_sav,NULL);
  2267.                         //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2268.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2269.                         back[i].status=0;    // READY
  2270.                         back[i].r.size=back[i].r.totalsize=back[i].range_req_size;
  2271.                         back[i].r.statuscode=304;     // NOT MODIFIED
  2272.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  2273.                           fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (good 416 message), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2274.                         }
  2275.                       }
  2276.                     }
  2277.                     
  2278.                     // transform 406 into 200 ; we'll catch embedded links inside the choice page
  2279.                     if (back[i].r.statuscode==406) {  // 'Not Acceptable'
  2280.                       back[i].r.statuscode=200;
  2281.                     }
  2282.  
  2283.                     // Various hacks to limit re-transfers when updating a mirror
  2284.                     // Force update if same size detected
  2285.                     if (opt->sizehack) {
  2286.                       // We already have the file
  2287.                       // and ask the remote server for an update
  2288.                       // Some servers, especially dynamic pages severs, always
  2289.                       // answer that the page has been modified since last visit
  2290.                       // And answer with a 200 (OK) response, and the same page
  2291.                       // If the size is the same, and the option has been set, we assume
  2292.                       // that the file is identical - and therefore let's break the connection
  2293.                       if (back[i].is_update) {          // mise α jour
  2294.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2295.                           htsblk r = cache_read(opt,cache,back[i].url_adr,back[i].url_fil,NULL,NULL);    // lire entrΘe cache
  2296.                           if (r.statuscode == 200) {  // OK pas d'erreur cache
  2297.                             LLint len1,len2;
  2298.                             len1=r.totalsize;
  2299.                             len2=back[i].r.totalsize;
  2300.                             if (r.size>0)
  2301.                               len1=r.size;
  2302.                             if (len1>0) {
  2303.                               if (len1 == len2) {             // tailles identiques
  2304.                                 back[i].r.statuscode=304;     // forcer NOT MODIFIED
  2305.                                 deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2306.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  2307.                                   fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2308.                                 }
  2309.                               }
  2310.                             }
  2311.                           } else {
  2312.                             if (opt->errlog!=NULL) {
  2313.                               fspc(opt->errlog,"warning"); fprintf(opt->errlog,"File seems complete (same size), but there was a cache read error: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2314.                             }
  2315.                           }
  2316.                           if (r.adr) {
  2317.                             freet(r.adr);
  2318.                           }
  2319.                         }
  2320.                       }
  2321.                     }
  2322.                     
  2323.                     // Various hacks to limit re-transfers when updating a mirror
  2324.                     // Detect already downloaded file (with another browser, for example)
  2325.                     if (opt->sizehack) {
  2326.                       if (!back[i].is_update) {          // mise α jour
  2327.                         if (back[i].r.statuscode==200 && !back[i].testmode) {  // 'OK'
  2328.                           if (!is_hypertext_mime(back[i].r.contenttype)) {    // not HTML
  2329.                             if (strnotempty(back[i].url_sav)) {  // target found
  2330.                               int size = fsize(back[i].url_sav);  // target size
  2331.                               if (size >= 0) {
  2332.                                 if (back[i].r.totalsize == size) {  // same size!
  2333.                                   deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2334.                                   back[i].status=0;    // READY
  2335.                                   back[i].r.size=back[i].r.totalsize;
  2336.                                   filenote(back[i].url_sav,NULL);
  2337.                                   //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2338.                                   back[i].r.statuscode=304;     // NOT MODIFIED
  2339.                                   if ((opt->debug>1) && (opt->log!=NULL)) {
  2340.                                     fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (same size file discovered), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2341.                                   }
  2342.                                 }
  2343.                               }
  2344.                             }
  2345.                           }
  2346.                         }
  2347.                       }
  2348.                     }
  2349.                     
  2350.                     // Various hacks to limit re-transfers when updating a mirror
  2351.                     // Detect bad range: header
  2352.                     if (opt->sizehack) {
  2353.                       // We have request for a partial file (with a 'Range: NNN-' header)
  2354.                       // and received a complete file notification (200), with 'Content-length: NNN'
  2355.                       // it might be possible that we had the complete file
  2356.                       // this is the case in *most* cases, so break the connection
  2357.                       if (back[i].r.is_write==0) {  // mode mΘmoire
  2358.                         if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  2359.                           if (!back[i].testmode) {  // pas mode test
  2360.                             if (strnotempty(back[i].url_sav)) {
  2361.                               if (strcmp(back[i].url_fil,"/robots.txt")) {
  2362.                                 if (back[i].r.statuscode==200) {  // 'OK'
  2363.                                   if (!is_hypertext_mime(back[i].r.contenttype)) {    // pas HTML
  2364.                                     if (back[i].r.statuscode==200) {      // "OK"
  2365.                                       if (back[i].range_req_size>0) {     // but Range: requested
  2366.                                         if (back[i].range_req_size == back[i].r.totalsize) {    // And same size
  2367. #if HTS_DEBUG_CLOSESOCK
  2368.                                           DEBUG_W("back_wait(skip_range): deletehttp\n");
  2369. #endif
  2370.                                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2371.                                           back[i].status=0;    // READY
  2372.                                           back[i].r.size=back[i].r.totalsize;
  2373.                                           filenote(back[i].url_sav,NULL);
  2374.                                           //xxusercommand(opt,0,NULL,back[i].url_sav,back[i].url_adr,back[i].url_fil);
  2375.                                           back[i].r.statuscode=304;     // NOT MODIFIED
  2376.                                           if ((opt->debug>1) && (opt->log!=NULL)) {
  2377.                                             fspc(opt->log,"debug"); fprintf(opt->log,"File seems complete (reget failed), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2378.                                           }
  2379.                                         }
  2380.                                       }
  2381.                                     }
  2382.                                     
  2383.                                   }
  2384.                                 }
  2385.                               }
  2386.                             }
  2387.                           }
  2388.                         }
  2389.                       }
  2390.                     }
  2391.                     // END - Various hacks to limit re-transfers when updating a mirror
  2392.  
  2393.                     /* 
  2394.                     End of status code and header-response hacks
  2395.                     */
  2396.  
  2397.                     
  2398.                     
  2399.                     /* Interdiction taille par le wizard? */
  2400.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2401.                       if (!back_checksize(opt,&back[i],1)) {
  2402.                         back[i].status=0;  // FINI
  2403.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2404.                         if (!back[i].testmode)
  2405.                           strcpybuff(back[i].r.msg,"File too big");
  2406.                         else
  2407.                           strcpybuff(back[i].r.msg,"Test: File too big");
  2408.                       }
  2409.                     }
  2410.                     
  2411.                     /* sinon, continuer */
  2412.                     /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok rΘcupΘrer body? */
  2413.                     // head: terminΘ
  2414.                     if (back[i].head_request) {
  2415.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  2416.                         fspc(opt->log,"debug"); fprintf(opt->log,"Tested file: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2417.                       }
  2418. #if HTS_DEBUG_CLOSESOCK
  2419.                       DEBUG_W("back_wait(head request): deletehttp\n");
  2420. #endif
  2421.                       // Couper connexion
  2422.                       if (!back[i].http11) {    /* NO KA */
  2423.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2424.                       }
  2425.                       back[i].status=0;  // terminΘ
  2426.                     }
  2427.                     // traiter une Θventuelle erreur 304 (cache α jour utilisable)
  2428.                     else if (back[i].r.statuscode==304) {  // document α jour dans le cache
  2429.                       // lire dans le cache
  2430.                       // ** NOTE: pas de vΘrif de la taille ici!!
  2431. #if HTS_DEBUG_CLOSESOCK
  2432.                       DEBUG_W("back_wait(file is not modified): deletehttp\n");
  2433. #endif
  2434.                       /* clear everything but connection: switch, close, and reswitch */
  2435.                       {
  2436.                         htsblk tmp;
  2437.                         memset(&tmp, 0, sizeof(tmp));
  2438.                         back_connxfr(&back[i].r, &tmp);
  2439.                         back[i].r=cache_read(opt,cache,back[i].url_adr,back[i].url_fil,back[i].url_sav,back[i].location_buffer);
  2440.                         back[i].r.location=back[i].location_buffer;
  2441.                         back_connxfr(&tmp,&back[i].r);
  2442.                       }
  2443.  
  2444.                       // hack:
  2445.                       // In case of 'if-unmodified-since' hack, a 304 status can be sent
  2446.                       // then, force 'ok' status
  2447.                       if (back[i].r.statuscode == -1) {
  2448.                         if (fexist(back[i].url_sav)) {
  2449.                           back[i].r.statuscode=200;     // OK
  2450.                           strcpybuff(back[i].r.msg, "OK (cached)");
  2451.                           back[i].r.is_file=1;
  2452.                           back[i].r.totalsize = back[i].r.size = fsize(back[i].url_sav);
  2453.                           get_httptype(back[i].r.contenttype, back[i].url_sav, 1);
  2454.                           if ((opt->debug>0) && (opt->log!=NULL)) {
  2455.                             fspc(opt->log,"debug"); fprintf(opt->log,"Not-modified status without cache guessed: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2456.                           }
  2457.                         }
  2458.                       }
  2459.  
  2460.                       // Status is okay?
  2461.                       if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture
  2462.                         back[i].status=0;         // OK prΩt
  2463.                         back[i].r.notmodified=1;  // NON modifiΘ!
  2464.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  2465.                           fspc(opt->log,"debug"); fprintf(opt->log,"File loaded after test from cache: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2466.                         }
  2467.  
  2468.                         // finalize
  2469.                         if (back[i].r.statuscode>0) {
  2470.                           back_finalize(opt,cache,back,i);
  2471.                         }
  2472.                         
  2473. #if DEBUGCA
  2474.                         printf("..document α jour aprΦs requΦte: %s%s\n",back[i].url_adr,back[i].url_fil);
  2475. #endif
  2476.                         
  2477.                         //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
  2478.                       } else {  // erreur
  2479.                         back[i].status=0;  // terminΘ
  2480.                         //printf("erreur cache\n");
  2481.                         
  2482.                       } 
  2483.                       
  2484. /********** NO - must complete the body! ********** */
  2485. #if 0
  2486.                     } else if ((back[i].r.statuscode==301)
  2487.                       || (back[i].r.statuscode==302)
  2488.                       || (back[i].r.statuscode==303)
  2489.                       || (back[i].r.statuscode==307)
  2490.                       || (back[i].r.statuscode==412)
  2491.                       || (back[i].r.statuscode==416)
  2492.                       ) {   // Ne pas prendre le html, erreurs connues et gΘrΘes
  2493. #if HTS_DEBUG_CLOSESOCK
  2494.                       DEBUG_W("back_wait(301,302,303,307,412,416..): deletehttp\n");
  2495. #endif
  2496.                       // Couper connexion
  2497.                       /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/
  2498.                       back_maydeletehttp(opt, back, back_max, i);
  2499.  
  2500.                       back[i].status=0;  // terminΘ
  2501.                       // finalize
  2502.                       if (back[i].r.statuscode>0) {
  2503.                         back_finalize(opt,cache,back,i);
  2504.                       }
  2505. #endif
  2506. /********** **************************** ********** */
  2507.                     } else {    // il faut aller le chercher
  2508.                       
  2509.                       // effacer buffer (requΦte)
  2510.                       if (!noFreebuff) {
  2511.                         deleteaddr(&back[i].r);
  2512.                         back[i].r.size=0;
  2513.                       }
  2514.                       
  2515.                       // traiter 206 (partial content)
  2516.                       // xxc SI CHUNK VERIFIER QUE CA MARCHE??
  2517.                       if (back[i].r.statuscode==206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
  2518.                         LLint sz=fsize(back[i].url_sav);
  2519. #if HDEBUG
  2520.                         printf("partial content: "LLintP" on disk..\n",(LLint)sz);
  2521. #endif
  2522.                         if (sz>=0) {
  2523.                           if (!is_hypertext_mime(back[i].r.contenttype)) {    // pas HTML
  2524.                             if (opt->getmode&2) {    // on peut ecrire des non html  **sinon ben euhh sera interceptΘ plus loin, donc rap sur ce qui va sortir**
  2525.                               filenote(back[i].url_sav,NULL);    // noter fichier comme connu
  2526.                               back[i].r.out=fopen(fconv(back[i].url_sav),"ab");  // append
  2527.                               if (back[i].r.out) {
  2528.                                 back[i].r.is_write=1;    // Θcrire
  2529.                                 back[i].r.size=sz;    // dΘja Θcrit
  2530.                                 back[i].r.statuscode=200;  // Forcer 'OK'
  2531.                                 if (back[i].r.totalsize>0)
  2532.                                   back[i].r.totalsize+=sz;    // plus en fait
  2533.                                 fseek(back[i].r.out,0,SEEK_END);  // α la fin
  2534. #if HDEBUG
  2535.                                 printf("continue interrupted file\n");
  2536. #endif
  2537.                               } else {    // On est dans la m**
  2538.                                 back[i].status=0;  // terminΘ (voir plus loin)
  2539.                                 strcpybuff(back[i].r.msg,"Can not open partial file");
  2540.                               }
  2541.                             }
  2542.                           } else {    // mΘmoire
  2543.                             FILE* fp=fopen(fconv(back[i].url_sav),"rb");
  2544.                             if (fp) {
  2545.                               LLint alloc_mem=sz + 1;
  2546.                               if (back[i].r.totalsize>0)
  2547.                                 alloc_mem+=back[i].r.totalsize;            // AJOUTER RESTANT!
  2548.                               if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) alloc_mem)) ) {
  2549.                                 back[i].r.size=sz;
  2550.                                 if (back[i].r.totalsize>0)
  2551.                                   back[i].r.totalsize+=sz;    // plus en fait
  2552.                                 if (( fread(back[i].r.adr,1,(INTsys)sz,fp)) != sz) {
  2553.                                   back[i].status=0;  // terminΘ (voir plus loin)
  2554.                                   strcpybuff(back[i].r.msg,"Can not read partial file");
  2555.                                 } else {
  2556.                                   back[i].r.statuscode=200;  // Forcer 'OK'
  2557. #if HDEBUG
  2558.                                   printf("continue in mem interrupted file\n");
  2559. #endif
  2560.                                 }
  2561.                               } else {
  2562.                                 back[i].status=0;  // terminΘ (voir plus loin)
  2563.                                 strcpybuff(back[i].r.msg,"No memory for partial file");
  2564.                               }
  2565.                               fclose(fp);
  2566.                             } else {  // Argh.. 
  2567.                               back[i].status=0;  // terminΘ (voir plus loin)
  2568.                               strcpybuff(back[i].r.msg,"Can not open partial file");
  2569.                             }
  2570.                           }
  2571.                         } else {    // Non trouvΘ??
  2572.                           back[i].status=0;  // terminΘ (voir plus loin)
  2573.                           strcpybuff(back[i].r.msg,"Can not find partial file");
  2574.                         }
  2575.                         // Erreur?
  2576.                         if (back[i].status==0) {
  2577.                           if (back[i].r.soc!=INVALID_SOCKET) {
  2578. #if HTS_DEBUG_CLOSESOCK
  2579.                             DEBUG_W("back_wait(206 solve problems): deletehttp\n");
  2580. #endif
  2581.                             deletehttp(&back[i].r);
  2582.                           }
  2583.                           back[i].r.soc=INVALID_SOCKET;
  2584.                           //back[i].r.statuscode=206;  ????????
  2585.                           back[i].r.statuscode=-5;
  2586.                           if (strnotempty(back[i].r.msg))
  2587.                             strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)");
  2588.                         }
  2589.                       }
  2590.                       
  2591.                       if (back[i].status!=0) {  // non terminΘ (erreur)
  2592.                         if (!back[i].testmode) {    // fichier normal
  2593.                           
  2594.                           if (back[i].r.empty && back[i].r.statuscode==200) {  // empty response
  2595.                             // Couper connexion
  2596.                             deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2597.                             back[i].status=0;  // terminΘ
  2598.                             if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((INTsys) 2)) ) {
  2599.                               back[i].r.adr[0] = 0;
  2600.                             }
  2601.                             back_finalize(opt,cache,back,i);
  2602.                           }
  2603.                           else if (!back[i].r.is_chunk) {    // pas de chunk
  2604.                             //if (back[i].r.http11!=2) {    // pas de chunk
  2605.                             back[i].is_chunk=0;
  2606.                             back[i].status=1;     // start body
  2607.                           } else {
  2608. #if CHUNKDEBUG==1
  2609.                             printf("[%d] chunk encoding detected %s..\n",(int)back[i].r.soc, back[i].url_fil);
  2610. #endif
  2611.                             back[i].is_chunk=1;
  2612.                             back[i].chunk_adr=NULL;
  2613.                             back[i].chunk_size=0;
  2614.                             back[i].chunk_blocksize=0;
  2615.                             back[i].status=98;    // start body wait chunk
  2616.                             back[i].r.totalsize=0;   /* devalidate size! (rfc) */
  2617.                           }
  2618.                           if (back[i].rateout>0) {
  2619.                             back[i].rateout_time=time_local();  // refresh pour transfer rate
  2620.                           }
  2621. #if HDEBUG
  2622.                           printf("(buffer) start body!\n");
  2623. #endif
  2624.                         } else {     // mode test, ne pas passer en 1!!
  2625.                           back[i].status=0;    // READY
  2626. #if HTS_DEBUG_CLOSESOCK
  2627.                           DEBUG_W("back_wait(test ok): deletehttp\n");
  2628. #endif
  2629.                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2630.                           if (back[i].r.statuscode==200) {
  2631.                             strcpybuff(back[i].r.msg,"Test: OK");
  2632.                             back[i].r.statuscode=-10;    // test rΘussi
  2633.                           }
  2634.                           else {    // test a ΘchouΘ, on ne change rien sauf que l'erreur est α titre indicatif
  2635.                             char tempo[1000];
  2636.                             strcpybuff(tempo,back[i].r.msg);
  2637.                             strcpybuff(back[i].r.msg,"Test: ");
  2638.                             strcatbuff(back[i].r.msg,tempo);
  2639.                           }
  2640.                           
  2641.                         }
  2642.                       }
  2643.                       
  2644.                       } 
  2645.                       
  2646.                       /*}*/
  2647.                       
  2648.                   }  // si LF
  2649.                 }  // r.size>2
  2650.               }  // si == 99
  2651.               
  2652.             } // si pas d'erreurs
  2653. #if BDEBUG==1
  2654.             printf("bytes overall: %d\n",back[i].r.size);
  2655. #endif
  2656.           }  // donnΘes dispo
  2657.           
  2658.           // en cas d'erreur cl, supprimer Θventuel fichier sur disque
  2659. #if HTS_REMOVE_BAD_FILES
  2660.           if (back[i].status<0) {
  2661.             if (!back[i].testmode) {    // pas en test
  2662.               remove(back[i].url_sav);    // Θliminer fichier (endommagΘ)
  2663.               //printf("&& %s\n",back[i].url_sav);
  2664.             }
  2665.           }
  2666. #endif
  2667.  
  2668.           /* funny log for commandline users */
  2669.           //if (!opt->quiet) {  
  2670.           // petite animation
  2671.           if (opt->verbosedisplay==1) {
  2672.             if (back[i].status==0) {
  2673.               if (back[i].r.statuscode==200)
  2674.                 printf("* %s%s ("LLintP" bytes) - OK"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size);
  2675.               else
  2676.                 printf("* %s%s ("LLintP" bytes) - %d"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size,back[i].r.statuscode);
  2677.               fflush(stdout);
  2678.             }
  2679.           }
  2680.           //}
  2681.           
  2682.  
  2683.       } // status>0
  2684.     }  // for
  2685.     
  2686.     // vΘrifier timeouts
  2687.     if (gestion_timeout) {
  2688.       TStamp act;
  2689.       act=time_local();    // temps en secondes
  2690.       for(i=0;i<back_max;i++) {
  2691.         if (back[i].status>0) {  // rΘception/connexion/..
  2692.           if (back[i].timeout>0) {
  2693.             //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
  2694.             if (((int) (act-back[i].timeout_refresh))>=back[i].timeout) {
  2695.               if (back[i].r.soc!=INVALID_SOCKET) {
  2696. #if HTS_DEBUG_CLOSESOCK
  2697.                 DEBUG_W("back_wait(timeout): deletehttp\n");
  2698. #endif
  2699.                 deletehttp(&back[i].r);
  2700.               }
  2701.               back[i].r.soc=INVALID_SOCKET;
  2702.               back[i].r.statuscode=-2;
  2703.               if (back[i].status==100)
  2704.                 strcpybuff(back[i].r.msg,"Connect Time Out");
  2705.               else if (back[i].status==101)
  2706.                 strcpybuff(back[i].r.msg,"DNS Time Out");
  2707.               else
  2708.                 strcpybuff(back[i].r.msg,"Receive Time Out");
  2709.               back[i].status=0;  // terminΘ
  2710.             } else if ((back[i].rateout>0) && (back[i].status<99)) {
  2711.               if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) {   // checker au bout de 15s
  2712.                 if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) {  // trop lent
  2713.                   back[i].status=0;  // terminΘ
  2714.                   if (back[i].r.soc!=INVALID_SOCKET) {
  2715. #if HTS_DEBUG_CLOSESOCK
  2716.                     DEBUG_W("back_wait(rateout): deletehttp\n");
  2717. #endif
  2718.                     deletehttp(&back[i].r);
  2719.                   }
  2720.                   back[i].r.soc=INVALID_SOCKET;
  2721.                   back[i].r.statuscode=-3;
  2722.                   strcpybuff(back[i].r.msg,"Transfer Rate Too Low");
  2723.                 }
  2724.               }
  2725.             }
  2726.           }
  2727.         }
  2728.       }
  2729.     }
  2730.     max_loop--;
  2731. #if HTS_ANALYSTE
  2732.     max_loop_chk++;
  2733. #endif
  2734.   } while((busy_state) && (busy_recv) && (max_loop>0));
  2735. #if HTS_ANALYSTE
  2736.   if ((!busy_recv) && (!busy_state)) {
  2737.     if (max_loop_chk>=1) {
  2738.       Sleep(10);    // un tite pause pour Θviter les lag..
  2739.     }
  2740.   }
  2741. #endif
  2742. }
  2743.  
  2744. int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize) {
  2745.   LLint size_to_test;
  2746.   if (check_only_totalsize)
  2747.     size_to_test=eback->r.totalsize;
  2748.   else
  2749.     size_to_test=max(eback->r.totalsize,eback->r.size);
  2750.   if (size_to_test>=0) {
  2751.     
  2752.     /* Interdiction taille par le wizard? */
  2753.     if (hts_testlinksize(opt,eback->url_adr,eback->url_fil,eback->r.totalsize/1024)==-1) {
  2754.       return 0;     /* interdit */
  2755.     }                     
  2756.     
  2757.     /* vΘrifier taille classique (heml et non html) */
  2758.     if ((istoobig(size_to_test,eback->maxfile_html,eback->maxfile_nonhtml,eback->r.contenttype))) {
  2759.       return 0;     /* interdit */
  2760.     }
  2761.   }
  2762.   return 1;
  2763. }
  2764.  
  2765. int back_checkmirror(httrackp* opt) {
  2766.   // Check max time
  2767.   if ((opt->maxsite>0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
  2768.     if (opt->errlog) {
  2769.       fprintf(opt->errlog,"More than "LLintP" bytes have been transfered.. giving up"LF,(LLint)opt->maxsite);
  2770.       test_flush;
  2771.     } 
  2772.     return 0;
  2773.   } else if ((opt->maxtime>0) && ((time_local()-HTS_STAT.stat_timestart)>opt->maxtime)) {            
  2774.     if (opt->errlog) {
  2775.       fprintf(opt->errlog,"More than %d seconds passed.. giving up"LF,opt->maxtime);
  2776.       test_flush;
  2777.     } 
  2778.     return 0;
  2779.   }
  2780.   return 1;   /* Ok, go on */
  2781. }
  2782.  
  2783.  
  2784. // octets transfΘrΘs + add
  2785. LLint back_transfered(LLint nb,lien_back* back,int back_max) {
  2786.   int i;
  2787.   // ajouter octets en instance
  2788.   for(i=0;i<back_max;i++)
  2789.     if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000))
  2790.       nb+=back[i].r.size;
  2791.   return nb;      
  2792. }
  2793.  
  2794. // infos backing
  2795. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  2796. void back_info(lien_back* back,int i,int j,FILE* fp) {
  2797.   if (back[i].status>=0) {
  2798.     char s[HTS_URLMAXSIZE*2+1024]; 
  2799.     s[0]='\0';
  2800.     back_infostr(back,i,j,s);
  2801.     strcatbuff(s,LF);
  2802.     fprintf(fp,"%s",s);
  2803.   }
  2804. }
  2805.  
  2806. // infos backing
  2807. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  2808. void back_infostr(lien_back* back,int i,int j,char* s) {
  2809.   if (back[i].status>=0) {
  2810.     int aff=0;
  2811.     if (j & 1) {
  2812.       if (back[i].status==100) {
  2813.         strcatbuff(s,"CONNECT ");
  2814.       } else if (back[i].status==99) {
  2815.         strcatbuff(s,"INFOS ");
  2816.         aff=1;
  2817.       } else if (back[i].status==98 || back[i].status==97) {
  2818.         strcatbuff(s,"INFOSC");             // infos chunk
  2819.         aff=1;
  2820.       }
  2821.       else if (back[i].status>0) {
  2822. #if HTS_ANALYSTE==2
  2823.         strcatbuff(s,"WAIT ");
  2824. #else
  2825.         strcatbuff(s,"RECEIVE "); 
  2826. #endif
  2827.         aff=1; 
  2828.       }
  2829.     } 
  2830.     if (j & 2) {
  2831.       if (back[i].status==0) {
  2832.         switch (back[i].r.statuscode) {
  2833.         case 200:
  2834.           strcatbuff(s,"READY ");
  2835.           aff=1;
  2836.           break;
  2837. #if HTS_ANALYSTE==2
  2838.         default:
  2839.           strcatbuff(s,"ERROR ");
  2840.           break;
  2841. #else
  2842.         case -1:
  2843.           strcatbuff(s,"ERROR ");
  2844.           aff=1;
  2845.           break;
  2846.         case -2:
  2847.           strcatbuff(s,"TIMEOUT ");
  2848.           aff=1;
  2849.           break;
  2850.         case -3:
  2851.           strcatbuff(s,"TOOSLOW ");
  2852.           aff=1;
  2853.           break;
  2854.         case 400:
  2855.           strcatbuff(s,"BADREQUEST ");
  2856.           aff=1;
  2857.           break;
  2858.         case 401: case 403:
  2859.           strcatbuff(s,"FORBIDDEN ");
  2860.           aff=1;
  2861.           break;
  2862.         case 404:
  2863.           strcatbuff(s,"NOT FOUND ");
  2864.           aff=1;
  2865.           break;
  2866.         case 500:
  2867.           strcatbuff(s,"SERVERROR ");
  2868.           aff=1;
  2869.           break;
  2870.         default:
  2871.           {
  2872.             char s2[256];
  2873.             sprintf(s2,"ERROR(%d)",back[i].r.statuscode);
  2874.             strcatbuff(s,s2);
  2875.           }
  2876.           aff=1;
  2877. #endif
  2878.         }
  2879.       }
  2880.     }
  2881.     
  2882.     if (aff) {
  2883.       {
  2884.         char s2[HTS_URLMAXSIZE*2+1024];
  2885.         sprintf(s2,"\"%s",back[i].url_adr); strcatbuff(s,s2);
  2886.         
  2887.         if (back[i].url_fil[0]!='/') strcatbuff(s,"/");
  2888.         sprintf(s2,"%s\" ",back[i].url_fil); strcatbuff(s,s2);
  2889.         sprintf(s,LLintP" "LLintP" ",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); strcatbuff(s,s2);
  2890.       }
  2891.     }
  2892.   }
  2893. }
  2894.  
  2895. // -- backing --
  2896.  
  2897. #undef test_flush
  2898.